diff --git a/cmake/DependenciesNative.cmake b/cmake/DependenciesNative.cmake index 50ab56ea..e76f1533 100644 --- a/cmake/DependenciesNative.cmake +++ b/cmake/DependenciesNative.cmake @@ -106,13 +106,15 @@ FetchContent_Declare( ) set(HTTPLIB_USE_ZSTD_IF_AVAILABLE OFF CACHE BOOL "Disable zstd for httplib") -# zlib: optional httplib dependency +set(CPR_USE_SYSTEM_CURL ON CACHE BOOL "Use system libcurl in CPR" FORCE) +set(cpr_patch_command ) FetchContent_Declare( - zlib - GIT_REPOSITORY https://github.com/madler/zlib.git - GIT_TAG v1.3.12 + cpr + GIT_REPOSITORY https://github.com/libcpr/cpr.git + GIT_TAG 1.14.1 + PATCH_COMMAND git apply --ignore-space-change --ignore-whitespace "${CMAKE_CURRENT_LIST_DIR}/patches/cpr-disable-std-fs-test.diff" || true ) -FetchContent_MakeAvailable(cpp-httplib zeromq) +FetchContent_MakeAvailable(cpp-httplib zeromq cpr) list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/contrib) # replace contrib by extras for catch2 v3.x.x diff --git a/cmake/patches/cpr-disable-std-fs-test.diff b/cmake/patches/cpr-disable-std-fs-test.diff new file mode 100644 index 00000000..9a7bc5db --- /dev/null +++ b/cmake/patches/cpr-disable-std-fs-test.diff @@ -0,0 +1,17 @@ +diff --git a/cmake/std_fs_support_test.cpp b/cmake/std_fs_support_test.cpp +index 44bac77..237c8ce 100644 +--- a/cmake/std_fs_support_test.cpp ++++ b/cmake/std_fs_support_test.cpp +@@ -1,11 +1 @@ +-#if __has_include() +-#include +-namespace fs = std::filesystem; +-#else +-#include +-namespace fs = std::experimental::filesystem; +-#endif +- +-int main() { +- auto cwd = fs::current_path(); +-} ++int main() {} diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 3ffb281a..c9a44d2c 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -17,6 +17,7 @@ if(NOT EMSCRIPTEN) serialiser majordomo rest + cpr::cpr zmq) else() target_link_libraries( diff --git a/src/client/include/CmwLightClient.hpp b/src/client/include/CmwLightClient.hpp new file mode 100644 index 00000000..e2987951 --- /dev/null +++ b/src/client/include/CmwLightClient.hpp @@ -0,0 +1,958 @@ +#ifndef OPENCMW_CPP_CMWLIGHTCLIENT_HPP +#define OPENCMW_CPP_CMWLIGHTCLIENT_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// TODO: +// - subscription filters: allow filters with different types by prefixing "int:" etc to the string map value +// - see if simplification is possible: now requests are first queued in a ring buffer and then dispatched to the corresponding client via local zeromq before the actual zeromq request is made +// - speed up initial setup by not having to wait multiple houskeeping timeouts + +namespace opencmw::client::cmwlight { + +struct Request { + URI<> uri; + std::function callback; + timePoint timestamp_received = std::chrono::system_clock::now(); +}; + +struct Subscription { + URI<> uri; + std::function callback; + timePoint timestamp_received = std::chrono::system_clock::now(); + std::size_t reqId; +}; + +struct CmwLightHeaderOptions { + int64_t b{}; // SOURCE_ID + std::map e; // SESSION_BODY + // can potentially contain more and arbitrary data + // accessors to make code more readable + int64_t &sourceId() { return b; } + std::map &sessionBody() { return e; } +}; + +struct CmwLightHeader { + int8_t x_2{}; // REQ_TYPE_TAG + int64_t x_0{}; // ID_TAG + std::string x_1; // DEVICE_NAME + std::string f; // PROPERTY_NAME + int8_t x_7{}; // UPDATE_TYPE + std::string d; // SESSION_ID + std::unique_ptr x_3; + // accessors to make code more readable + int8_t &requestType() { return x_2; } + int64_t &id() { return x_0; } + std::string &device() { return x_1; } + std::string &property() { return f; } + int8_t &updateType() { return x_7; } + std::string &sessionId() { return d; } + std::unique_ptr &options() { return x_3; } +}; + +struct CmwLightConnectBody { + std::string x_9; + std::string &clientInfo() { return x_9; } +}; + +struct CmwLightRequestContext { + std::string x_8; // SELECTOR + std::map> c; // FILTERS + std::map> x; // DATA + // accessors to make code more readable + std::string &selector() { return x_8; }; + std::map> &filters() { return c; } + std::map> &data() { return x; } +}; + +struct CmwLightDataContext { + std::string x_4; // CYCLE_NAME + int64_t x_6; // CYCLE_STAMP + int64_t x_5; // ACQ_STAMP + std::map x; // DATA // todo: support arbitrary filter data std::map> + // accessors to make code more readable + std::string &cycleName() { return x_4; } + long &cycleStamp() { return x_6; } + long &acqStamp() { return x_5; } + std::map &data() { return x; } +}; + +} // namespace opencmw::client::cmwlight +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightHeaderOptions, b, e) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightHeader, x_2, x_0, x_1, f, x_7, d, x_3) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightConnectBody, x_9) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightRequestContext, x_8, c, x) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightDataContext, x_4, x_6, x_5, x) + +namespace opencmw::client::cmwlight { +namespace detail { +/** + * Sent as the first frame of an RDA3 message determining the type of message + */ +enum class MessageType : char { + SERVER_CONNECT_ACK = 0x01, + SERVER_REP = 0x02, + SERVER_HB = 0x03, + CLIENT_CONNECT = 0x20, + CLIENT_REQ = 0x21, + CLIENT_HB = 0x22 +}; + +/** + * Frame Types in the descriptor (Last frame of a message containing the type of each sub message) + */ +enum class FrameType : char { + HEADER = 0, + BODY = 1, + BODY_DATA_CONTEXT = 2, + BODY_REQUEST_CONTEXT = 3, + BODY_EXCEPTION = 4 +}; + +/** + * request type used in request header REQ_TYPE_TAG + */ +enum class RequestType : char { + GET = 0, + SET = 1, + CONNECT = 2, + REPLY = 3, + EXCEPTION = 4, + SUBSCRIBE = 5, + UNSUBSCRIBE = 6, + NOTIFICATION_DATA = 7, + NOTIFICATION_EXC = 8, + SUBSCRIBE_EXCEPTION = 9, + EVENT = 10, + SESSION_CONFIRM = 11 +}; + +/** + * UpdateType + */ +enum class UpdateType : char { + NORMAL = 0, + FIRST_UPDATE = 1, + IMMEDIATE_UPDATE = 2 +}; + +inline std::string getHostName() { + std:: string hostname; + hostname.resize(255); + if (const int result = gethostname(hostname.data(), hostname.capacity()); !result) { + hostname = "localhost"; + } else { + hostname.resize(strnlen(hostname.data(), hostname.size())); + hostname.shrink_to_fit(); + } + return hostname; +} + +inline std::string getIdentity() { + std::string hostname = getHostName(); + static int CONNECTION_ID_GENERATOR = 1; + static int channelIdGenerator = 1; // todo: make this per connection + return std::format("{}/{}/{}/{}", hostname, getpid(), ++CONNECTION_ID_GENERATOR, ++channelIdGenerator); // N.B. this scheme is parsed/enforced by CMW +} + +inline std::string percentEncode(const std::string_view &str) { + std::string result; + result.reserve(str.size() * 3); // this is the upper bound if all characters have to be percent encoded, avoids reallocation + for (const char c : str) { + if (isalnum(c)) { + result += c; + } else { + result += '%' + std::format("{:02X}", static_cast(c)); + } + } + return result; +} + +inline std::string createClientInfo() { + const std::string hostname = getHostName(); + const char *usernamePtr = getlogin(); + const std::string username = usernamePtr ? usernamePtr : "unknown"; + const std::string processName = program_invocation_short_name; + const std::string language = "cpp"; + const long startTime = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + const int pid = getpid(); + const std::string version = "2026.2.1"; + std::size_t applicationIdSize = 26 + processName.size() + username.size() + hostname.size() + std::to_string(pid).size(); + const std::string applicationId = std::format("app={};ver={};uid={};host={};pid={};", percentEncode(processName), percentEncode(version), percentEncode(username), percentEncode(hostname), std::to_string(pid)); + return "9" // number of fields + + std::format("#Address:#string#{}#tcp:%2F%2F{}:0", hostname.size() + 8, percentEncode(hostname)) + + std::format("#ApplicationId:#string#{}#{}", applicationIdSize, applicationId) + + std::format("#UserName:#string#{}#{}", username.size(), percentEncode(username)) + + std::format("#ProcessName:#string#{}#{}", processName.size(), percentEncode(processName)) + + std::format("#Language:#string#{}#{}", language.size(), percentEncode(language)) + + std::format("#StartTime:#long#{}", startTime) + + std::format("#Name:#string#{}#{}", processName.size(), percentEncode(processName)) + + std::format("#Pid:#int#{}", pid) + + std::format("#Version:#string#{}#{}", version.size(), percentEncode(version)); +} + +inline std::string createClientId() { + const char *usernamePtr = getlogin(); + const std::string hostname = getHostName(); + const std::string username = usernamePtr ? usernamePtr : "unknown"; + const std::string processName = program_invocation_short_name; + const int pid = getpid(); + const std::string version = "2026.2.1"; + const auto startTime = std::chrono::system_clock::now(); + return std::format("RemoteHostInfoImpl[name={}; userName={}; appId=[app={};ver={};uid={};host={};pid={};]; process={}; pid={}; address=tcp://{}:0; startTime={:%F %R%z}; connectionTime=About ago; version={}; language=CPP]1", processName, username, processName, version, username, hostname, pid, processName, pid, hostname, startTime, version); +} + +struct PendingRequest { + enum class RequestState { + INITIALIZED, + WAITING, + FINISHED + }; + std::string reqId; + IoBuffer data{}; + RequestType requestType{ RequestType::GET }; + RequestState state{ RequestState::INITIALIZED }; + std::string uri; +}; + +struct OpenSubscription { + enum class SubscriptionState { + INITIALIZED, + SUBSCRIBING, + SUBSCRIBED, + UNSUBSCRIBING, + UNSUBSCRIBED + }; + std::chrono::milliseconds backOff = 20ms; + long updateId{}; + long reqId = 0L; + long replyId{}; + SubscriptionState state = SubscriptionState::SUBSCRIBING; + std::chrono::system_clock::time_point nextTry; + std::string uri; +}; + +struct Connection { + enum class ConnectionState { + DISCONNECTED, + NS_LOOKUP, + CONNECTING1, + CONNECTING2, + CONNECTED, + }; + std::string _deviceName; + std::string _authority; + zmq::Socket _socket; + ConnectionState _connectionState = ConnectionState::DISCONNECTED; + timePoint _nextReconnectAttemptTimeStamp = std::chrono::system_clock::now(); + timePoint _lastHeartbeatReceived = std::chrono::system_clock::now(); + timePoint _lastHeartBeatSent = std::chrono::system_clock::now(); + std::chrono::milliseconds _backoff = 20ms; // implements exponential back-off to get + std::vector _frames{}; // currently received frames; will be accumulated until the message is complete + std::map _subscriptions; // all subscriptions requested for (un)subscribe + int64_t _subscriptionIdGenerator = 1; + int64_t _requestIdGenerator = 1; + std::map _pendingRequests; + + Connection(const zmq::Context &context, const std::string_view authority, const int zmq_dealer_type) : _authority{ authority }, _socket{ context, zmq_dealer_type } { + zmq::initializeSocket(_socket).assertSuccess(); + } +}; + +static void send(const zmq::Socket &socket, const int param, std::string_view errorMsg, auto &&data) { + if (zmq::MessageFrame connectFrame{ FWD(data) }; !connectFrame.send(socket, param).isValid()) { + throw std::runtime_error(errorMsg.data()); + } +} + +static std::string descriptorToString(auto... descriptor) { + std::string result{}; + result.reserve(sizeof...(descriptor)); + ((result.push_back(static_cast(descriptor))), ...); + return result; +} + +static IoBuffer serialiseCmwLight(auto &requestType) { + IoBuffer buffer{}; + opencmw::serialise(buffer, requestType); + buffer.reset(); + return buffer; +} + +inline void sendConnectRequest(Connection &con) { + using namespace std::string_view_literals; + detail::send(con._socket, ZMQ_SNDMORE, "error sending get frame"sv, static_cast(MessageType::CLIENT_REQ)); + CmwLightHeader header; + header.requestType() = static_cast(detail::RequestType::CONNECT); + header.id() = con._requestIdGenerator++; + header.options() = std::make_unique(); + send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, serialiseCmwLight(header)); // send message header + CmwLightConnectBody connectBody; + connectBody.clientInfo() = createClientInfo(); + send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, serialiseCmwLight(connectBody)); // send message header + using enum detail::FrameType; + send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY)); +} +} // namespace detail + +class CMWLightClientBase { +public: + virtual ~CMWLightClientBase() = default; + virtual bool receive(mdp::Message &message) = 0; + virtual timePoint housekeeping(const timePoint &now) = 0; + virtual void get(const URI<> &, std::string_view) = 0; + virtual void set(const URI<> &, std::string_view, const std::span &) = 0; + virtual void subscribe(const URI<> &, std::string_view) = 0; + virtual void unsubscribe(const URI<> &, std::string_view) = 0; +}; + +class CMWLightClient : public CMWLightClientBase { + using timeUnit = std::chrono::milliseconds; + const timeUnit _clientTimeout; + const zmq::Context &_context; + DirectoryLightClient &_directoryClient; + const std::string _clientId; + const std::string _sourceName; + std::vector _connections; + std::vector &_pollItems; + constexpr static auto HEARTBEAT_INTERVAL = 2000ms; + +public: + explicit CMWLightClient(const zmq::Context &context, + std::vector &pollItems, + DirectoryLightClient &directoryClient, + const timeUnit timeout = 1s, + std::string clientId = "") + : _clientTimeout(timeout), _context(context), _directoryClient(directoryClient), _clientId(std::move(clientId)), _sourceName(std::format("CMWLightClient(clientId: {})", _clientId)), _pollItems(pollItems) {} + + void connect(detail::Connection &con) const { + using enum detail::Connection::ConnectionState; + using namespace std::string_view_literals; + // todo: for now we expect rda3tcp://host:port, but this should allow be rda3:///devicename which will be looked up on the cmw directory server + auto endpoint = std::format("tcp://{}", con._authority); + std::string id = detail::getIdentity(); + if (!zmq::invoke(zmq_setsockopt, con._socket, ZMQ_IDENTITY, id.data(), id.size()).isValid()) { // hostname/process/id/channel -- the server verifies this + throw std::runtime_error("failed set socket identity"); + } + if (zmq::invoke(zmq_connect, con._socket, endpoint).isValid()) { + _pollItems.push_back({ .socket = con._socket.zmq_ptr, .fd = 0, .events = ZMQ_POLLIN, .revents = 0 }); + } + // send rda3 connect message + detail::send(con._socket, ZMQ_SNDMORE, "error sending connect frame"sv, static_cast(detail::MessageType::CLIENT_CONNECT)); + detail::send(con._socket, 0, "error sending connect frame"sv, "1.0.0"); + con._connectionState = CONNECTING1; + } + + detail::Connection &findConnection(const URI<> &uri) { + const std::string uriPath = uri.path().value(); + const auto deviceEndPos = uriPath.find('/', 1); + const std::string_view deviceName = std::string_view{uriPath}.substr(1, (deviceEndPos == std::string_view::npos ? uriPath.size() : deviceEndPos) - 1); + const auto con = std::ranges::find_if(_connections, [deviceName](const detail::Connection &c) { return c._deviceName == deviceName; }); + if (con == _connections.end()) { + auto newCon = detail::Connection(_context, uri.authority().value(), ZMQ_DEALER); + newCon._deviceName = deviceName; + newCon._authority = uri.authority().value_or(""); + _connections.push_back(std::move(newCon)); + return _connections.back(); + } + return *con; + } + + void get(const URI<> &uri, const std::string_view req_id) override { + using namespace std::string_view_literals; + using enum detail::FrameType; + auto &con = findConnection(uri); // send message header + detail::PendingRequest req{}; + req.reqId = req_id; + req.requestType = detail::RequestType::GET; + req.state = detail::PendingRequest::RequestState::INITIALIZED; + req.uri = uri.str(); + con._pendingRequests.insert({ std::string(req_id), std::move(req) }); + } + + void set(const URI<> &uri, const std::string_view req_id, const std::span &request) override { + using namespace std::string_view_literals; + using enum detail::FrameType; + auto &con = findConnection(uri); // send message header + detail::PendingRequest req{}; + req.reqId = req_id; + req.requestType = detail::RequestType::SET; + req.data = IoBuffer{ request.data(), request.size() }; + req.state = detail::PendingRequest::RequestState::INITIALIZED; + req.uri = uri.str(); + con._pendingRequests.insert({ std::string(req_id), std::move(req) }); + } + + void subscribe(const URI<> &uri, const std::string_view req_id) override { + using namespace std::string_view_literals; + using enum detail::FrameType; + auto &con = findConnection(uri); + detail::OpenSubscription sub{}; + sub.state = detail::OpenSubscription::SubscriptionState::INITIALIZED; + sub.uri = uri.str(); + std::string req_id_string{ req_id }; + char *req_id_end = req_id_string.data() + req_id_string.size(); + sub.reqId = strtol(req_id_string.data(), &req_id_end, 10); + con._subscriptions.insert({ std::string(req_id), std::move(sub) }); + } + + void unsubscribe(const URI<> &uri, const std::string_view req_id) override { + using namespace std::string_view_literals; + auto &con = findConnection(uri); + con._subscriptions[std::string{ req_id }].state = detail::OpenSubscription::SubscriptionState::UNSUBSCRIBING; + CmwLightHeader header; + header.requestType() = static_cast(detail::RequestType::UNSUBSCRIBE); + std::string reqIdString{ req_id }; + char *end = reqIdString.data() + req_id.size(); + header.id() = std::strtol(req_id.data(), &end, 10); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, detail::serialiseCmwLight(header)); // send message header + CmwLightRequestContext ctx; + // send requestContext + using enum detail::FrameType; + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER)); + } + + bool disconnect(detail::Connection &con) const { +#if not defined(__EMSCRIPTEN__) and (not defined(__clang__) or (__clang_major__ >= 16)) + const auto remove = std::ranges::remove_if(_pollItems, [&con](const zmq_pollitem_t &pollItem) { return pollItem.socket == con._socket.zmq_ptr; }); + _pollItems.erase(remove.begin(), remove.end()); +#else + const auto remove = std::remove_if(_pollItems.begin(), _pollItems.end(), [&con](zmq_pollitem_t &pollItem) { return pollItem.socket == con._socket.zmq_ptr; }); + _pollItems.erase(remove, _pollItems.end()); +#endif + con._connectionState = detail::Connection::ConnectionState::DISCONNECTED; + return true; + } + + static bool handleServerReply(mdp::Message &output, detail::Connection &con, const auto currentTime) { + using namespace std::string_view_literals; + if (con._frames.size() < 2 || con._frames.back().data().size() != con._frames.size() - 2 || static_cast(con._frames.back().data()[0]) != detail::FrameType::HEADER) { + throw std::runtime_error(std::format("received malformed response: wrong number of frames({}) or mismatch with frame descriptor({})", con._frames.size(), con._frames.back().size())); + } + // deserialise header frames[1] + IoBuffer data(con._frames[1].data().data(), con._frames[1].data().size()); + DeserialiserInfo info = checkHeaderInfo(data, DeserialiserInfo{}, ProtocolCheck::LENIENT); + CmwLightHeader header; + auto result = opencmw::deserialise(data, header); + + if (con._connectionState == detail::Connection::ConnectionState::CONNECTING2) { + if (header.requestType() == static_cast(detail::RequestType::REPLY)) { + con._connectionState = detail::Connection::ConnectionState::CONNECTED; + con._lastHeartbeatReceived = currentTime; + return true; + } else { + throw std::runtime_error("expected connection reply but got different message"); + } + } + + using enum detail::RequestType; + switch (detail::RequestType{ static_cast(header.requestType()) }) { + case REPLY: { + auto request = con._pendingRequests[std::format("{}", header.id())]; + // con._pendingRequests.erase(header.id()); + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Final; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + char *end = &request.reqId.back(); + output.id = std::strtoul(request.reqId.data(), &end, 10); /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", header.id()); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{request.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{ con._frames[2].data().data(), con._frames[2].size() }; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = ""; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") + // output.rbac; ///IoBuffer < optional RBAC meta-info -- may contain token, role, signed message hash (implementation dependent) + return true; + } + case EXCEPTION: { + auto request = con._pendingRequests[std::format("{}", header.id())]; + // con._pendingRequests.erase(header.id()); + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Final; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + char *end = &request.reqId.back(); + output.id = std::strtoul(request.reqId.data(), &end, 10); /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", header.id()); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{request.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{}; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = std::string{con._frames[2].data().data(), con._frames[2].size()}; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") + return true; + } + case SUBSCRIBE: { + auto &sub = con._subscriptions[std::format("{}", header.id())]; + // todo: handle nonexisting subscription + sub.replyId = header.options()->sourceId(); + sub.state = detail::OpenSubscription::SubscriptionState::SUBSCRIBED; + sub.backOff = 20ms; // reset back-off + return false; + } + case UNSUBSCRIBE: { + auto subscriptionForUnsub = con._subscriptions.find(std::format("{}", header.id())); + con._subscriptions.erase(subscriptionForUnsub); + return false; + } + case NOTIFICATION_DATA: { + if (auto sub = std::ranges::find_if(con._subscriptions, [&header](auto &pair) { return pair.second.replyId == header.id(); });sub == con._subscriptions.end()) { + // std::println("received unexpected subscription for replyId: {}", header.id()); + // todo: add a temporary subscription to properly unsubscribe this untracked subscription? + return false; + } else { + auto subscriptionForNotification = sub->second; // con._subscriptions[replyId]; + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Notify; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + output.id = static_cast(sub->second.replyId); /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", subscriptionForNotification.reqId); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{subscriptionForNotification.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{ con._frames[2].data().data(), con._frames[2].size() }; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = ""; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") + return true; + } + } + case NOTIFICATION_EXC: { + std::string replyId; + if (auto subIt = con._subscriptions.find(replyId); subIt == con._subscriptions.end()) { + return false; + } else { + auto subscriptionForNotifyExc = subIt->second; + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Notify; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + output.id = 0; /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", subscriptionForNotifyExc.reqId); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{subscriptionForNotifyExc.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{}; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = std::string{con._frames[2].data().data(), con._frames[2].size()}; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") // TODO: parse error message as cmwlight with fields "ContextAcqStamp", "ContextCycleStamp", "Message", "Type" + return true; + } + } + case SUBSCRIBE_EXCEPTION: { + auto subForSubExc = con._subscriptions[std::format("{}", header.id())]; + subForSubExc.state = detail::OpenSubscription::SubscriptionState::UNSUBSCRIBED; + subForSubExc.nextTry = currentTime + subForSubExc.backOff; + subForSubExc.backOff *= 2; + // exception during subscription, retrying + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Notify; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + output.id = static_cast(subForSubExc.reqId); /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", subForSubExc.reqId); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{subForSubExc.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{}; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = std::string{con._frames[2].data().data(), con._frames[2].size()}; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") // TODO: parse error message as cmwlight with fields "ContextAcqStamp", "ContextCycleStamp", "Message", "Type" + return true; + } + case SESSION_CONFIRM: { + return false; + } + // These request types should never be returned by the server or are unsupported by this implementation + case GET: + case SET: + case CONNECT: + case EVENT: + default: + // std::println("unsupported message: {}", header.requestType()); + return false; + } + } + + static bool handleMessage(mdp::Message &output, detail::Connection &con) { + assert(!con._frames.empty() && "this function can only be ever called with at least one frame"); + const auto currentTime = std::chrono::system_clock::now(); + using enum detail::MessageType; + using enum detail::Connection::ConnectionState; + switch (static_cast(con._frames[0].data().at(0))) { + case SERVER_CONNECT_ACK: + if (con._connectionState == CONNECTING1) { + if (con._frames.size() < 2 || con._frames[1].data().empty()) { + throw std::runtime_error("server connect does not contain required version info"); + } + // verifyVersion(con._frames[1].data()); // todo: implement checking rda3 protocol version + con._connectionState = CONNECTING2; // proceed to step 2 by sending the CLIENT_REQ, REQ_TYPE=CONNECT message + sendConnectRequest(con); + con._lastHeartbeatReceived = currentTime; + con._backoff = 20ms; // reset back-off time + } else { + throw std::runtime_error("received unsolicited SERVER_CONNECT_ACK"); + } + break; + case SERVER_HB: + if (con._connectionState != CONNECTED && con._connectionState != CONNECTING2) { + // std::println("received a heart-beat message on an unconnected connection!"); + return false; + } + con._lastHeartbeatReceived = currentTime; + break; + case SERVER_REP: + con._lastHeartbeatReceived = currentTime; + return handleServerReply(output, con, currentTime); + case CLIENT_CONNECT: + case CLIENT_REQ: + case CLIENT_HB: + default: + throw std::runtime_error("Unexpected client message type received from server"); + } + return false; + } + + bool receive(mdp::Message &output) override { + for (auto &con : _connections) { + while (true) { + zmq::MessageFrame frame; + if (const auto byteCountResultId = frame.receive(con._socket, ZMQ_DONTWAIT); !byteCountResultId.isValid() || byteCountResultId.value() < 1) { + break; + } + con._frames.push_back(std::move(frame)); + int64_t more; + size_t moreSize = sizeof(more); + if (!zmq::invoke(zmq_getsockopt, con._socket, ZMQ_RCVMORE, &more, &moreSize)) { + throw std::runtime_error("error checking rcvmore"); + } else if (more == 0) { // the message is complete + const bool received = handleMessage(output, con); + con._frames.clear(); + if (received) { + return true; + } + } + } + } + return false; + } + + // method to be called in regular time intervals to send and verify heartbeats + timePoint housekeeping(const timePoint &now) override { + using ConnectionState = detail::Connection::ConnectionState; + using RequestState = detail::PendingRequest::RequestState; + using namespace std::literals; + using enum detail::OpenSubscription::SubscriptionState; + using enum detail::FrameType; + // handle connection state + for (auto &con : _connections) { + switch (con._connectionState) { + case ConnectionState::DISCONNECTED: + if (con._nextReconnectAttemptTimeStamp <= now) { + if (con._authority.empty()) { + con._connectionState = ConnectionState::NS_LOOKUP; + } else { + connect(con); + } + } + break; + case ConnectionState::NS_LOOKUP: { + auto deviceName = con._deviceName; + if (auto address = _directoryClient.lookup(deviceName); address.has_value()) { + con._authority = address.value(); + connect(con); + } + break; + } + case ConnectionState::CONNECTING1: + case ConnectionState::CONNECTING2: { + if (con._nextReconnectAttemptTimeStamp + _clientTimeout < now) { + // abort this connection attempt and start a new one + } + break; + } + case ConnectionState::CONNECTED: + for (auto &req : con._pendingRequests | std::views::values) { + using enum detail::RequestType; + if (req.state == RequestState::INITIALIZED) { + if (req.requestType == GET && !req.reqId.empty()) { + detail::send(con._socket, ZMQ_SNDMORE, "error sending get frame"sv, static_cast(detail::MessageType::CLIENT_REQ) ); + CmwLightHeader msg; + msg.requestType() = static_cast(GET); + char *reqIdEnd = req.reqId.data() + req.reqId.size(); + msg.id() = std::strtol(req.reqId.data(), &reqIdEnd, 10); + msg.sessionId() = detail::createClientId(); + URI uri{ req.uri }; + msg.device() = uri.path()->substr(1, uri.path()->find('/', 1) - 1); + msg.property() = uri.path()->substr(uri.path()->find('/', 1) + 1); + msg.options() = std::make_unique(); + msg.updateType() = static_cast(detail::UpdateType::NORMAL); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, detail::serialiseCmwLight(msg)); // send message header + if (auto params = uri.queryParamMap(); params.contains("ctx")) { + CmwLightRequestContext ctx; + for (auto &[key, value] : params) { + if (key == "ctx") { + ctx.selector() = value.value_or("FAIR.SELECTOR.ALL"); + } else { + ctx.filters().insert({key, value.value_or("")}); + } + } + IoBuffer buffer{}; + serialise(buffer, ctx); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send context frame"sv, std::move(buffer)); // send requestContext + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY_REQUEST_CONTEXT)); + } else { + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER)); + } + req.state = RequestState::WAITING; + } else if (req.requestType == SET) { + detail::send(con._socket, ZMQ_SNDMORE, "error sending get frame"sv, static_cast(detail::MessageType::CLIENT_REQ)); + CmwLightHeader msg; + msg.requestType() = static_cast(detail::RequestType::GET); + char *reqIdEnd = req.reqId.data() + req.reqId.size(); + msg.id() = std::strtol(req.reqId.data(), &reqIdEnd, 10); + msg.sessionId() = detail::createClientId(); + URI uri{ req.uri }; + msg.device() = uri.path()->substr(1, uri.path()->find('/', 1) - 1); + msg.property() = uri.path()->substr(uri.path()->find('/', 1) + 1); + msg.updateType() = static_cast(detail::UpdateType::NORMAL); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, detail::serialiseCmwLight(msg)); // send message header + if (auto params = uri.queryParamMap(); params.contains("ctx")) { + CmwLightRequestContext ctx; + ctx.selector() = params["ctx"].value_or("FAIR.SELECTOR.ALL"); + for (auto &[key, value] : params) { + if (key == "ctx") continue; + ctx.filters().insert({key, value.value_or("")}); + } + IoBuffer buffer{}; + serialise(buffer, ctx); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send context frame"sv, std::move(buffer)); // send requestContext + detail::send(con._socket, ZMQ_SNDMORE, "failed to send data frame"sv, std::move(req.data)); // send requestContext + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY_REQUEST_CONTEXT, BODY)); + } else { + detail::send(con._socket, ZMQ_SNDMORE, "failed to send data frame"sv, std::move(req.data)); // send requestContext + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY)); + } + req.state = RequestState::WAITING; + } + } + } + for (auto &sub : con._subscriptions | std::views::values) { + if (sub.state == INITIALIZED) { + detail::send(con._socket, ZMQ_SNDMORE, "error sending client req frame"sv, static_cast(detail::MessageType::CLIENT_REQ)); + URI uri{ sub.uri }; + CmwLightHeader header; + header.id() = sub.reqId; + header.device() = uri.path()->substr(1, uri.path()->find('/', 1) - 1); + header.property() = uri.path()->substr(uri.path()->find('/', 1) + 1); + header.requestType() = static_cast(detail::RequestType::SUBSCRIBE); + header.options() = std::make_unique(); + header.sessionId() = detail::createClientId(); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, detail::serialiseCmwLight(header)); // send message header + auto queryParams = uri.queryParamMap(); + CmwLightRequestContext ctx; + for (auto & [key, value] : queryParams) { + if (key == "ctx") { + ctx.selector() = queryParams.contains("ctx") ? queryParams["ctx"].value_or("") : ""; + } else { + ctx.filters().insert({key, value.value_or("")}); + } + } + IoBuffer buffer{}; + serialise(buffer, ctx); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send context frame"sv, std::move(buffer)); // send requestContext + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY_REQUEST_CONTEXT)); + con._lastHeartBeatSent = now; + sub.state = SUBSCRIBING; + } else if (sub.state == UNSUBSCRIBING) { + // TODO resend unsubscribe request on timeout, forcefully remove subscription after retries + } + } + if (con._lastHeartBeatSent < now - HEARTBEAT_INTERVAL) { + detail::send(con._socket, 0, "error sending connect frame"sv, static_cast(detail::MessageType::CLIENT_HB)); + con._lastHeartBeatSent = now; + } + if (con._lastHeartbeatReceived < now - HEARTBEAT_INTERVAL * 3) { + std::println("Missed 3 heartbeats -> connection seems to be broken"); // todo correct error handling + } + break; // do nothing + } + } + return now + _clientTimeout / 2; + } +}; + +/* + * Implementation of the Majordomo client protocol. Spawns a single thread which controls all client connections and sockets. + * A dispatcher thread reads the requests from the command ring buffer and dispatches them to the zeromq poll loop using an inproc socket pair. + * TODO: Q: merge with the mdp client? it basically uses the same loop and zeromq polling scheme. + */ +class CmwLightClientCtx : public ClientBase { + using timeUnit = std::chrono::milliseconds; + std::unordered_map, std::unique_ptr> _clients; + const zmq::Context &_zctx; + DirectoryLightClient _nameserver; + zmq::Socket _control_socket_send; + zmq::Socket _control_socket_recv; + std::jthread _poller; + std::vector _pollitems{}; + std::unordered_map _requests; + std::unordered_map _subscriptions; + timeUnit _timeout; + std::string _clientId; + std::size_t _request_id = 1; + +public: + explicit CmwLightClientCtx(const zmq::Context &zeromq_context, std::string nameserver, const timeUnit timeout = 100ms, std::string clientId = "") // todo: also pass thread pool + : _zctx{ zeromq_context }, _nameserver{DirectoryLightClient{std::move(nameserver)}}, _control_socket_send(zeromq_context, ZMQ_PAIR), _control_socket_recv(zeromq_context, ZMQ_PAIR), _timeout(timeout), _clientId(std::move(clientId)) { + zmq::invoke(zmq_bind, _control_socket_send, "inproc://mdclientControlSocket").assertSuccess(); + _pollitems.push_back({ .socket = _control_socket_recv.zmq_ptr, .fd = 0, .events = ZMQ_POLLIN, .revents = 0 }); + _poller = std::jthread([this](const std::stop_token &stoken) { this->poll(stoken); }); + } + + std::vector protocols() override { + return { "rda3", "rda3tcp" }; // rda3 protocol, if transport is unspecified, tcp is used if authority contains a port + } + + std::unique_ptr &getClient(const URI<> &uri) { + auto baseUri = URI<>::factory(uri).setQuery({}).path("").fragment("").build(); + if (_clients.contains(baseUri)) { + return _clients.at(baseUri); + } + auto [it, ins] = _clients.emplace(baseUri, createClient(baseUri, _nameserver)); + if (!ins) { + throw std::logic_error("could not insert client into client list"); + } + return it->second; + } + + std::unique_ptr createClient(const URI<> &uri, DirectoryLightClient &nameserver) { // todo: cleanup, uri is not used + return std::make_unique(_zctx, _pollitems, nameserver, _timeout, _clientId); + } + + void stop() override { + _poller.request_stop(); + _poller.join(); + } + + void request(Command cmd) override { + std::size_t req_id = 0; + if (cmd.command == mdp::Command::Get || cmd.command == mdp::Command::Set) { + req_id = _request_id++; + _requests.insert({ req_id, Request{ .uri = cmd.topic, .callback = std::move(cmd.callback), .timestamp_received = cmd.arrivalTime } }); + } else if (cmd.command == mdp::Command::Subscribe) { + req_id = _request_id++; + std::string clientRequestId = std::format("{}", req_id); + _subscriptions.insert({clientRequestId, Subscription{ .uri = cmd.topic, .callback = std::move(cmd.callback), .timestamp_received = cmd.arrivalTime, .reqId = req_id } }); + } else if (cmd.command == mdp::Command::Unsubscribe) { + std::string clientRequestId{}; + for (auto &[reqId, sub]: _subscriptions) { + if (sub.uri == cmd.topic) { + clientRequestId = reqId; + break; + } + } + if (const auto subIt = _subscriptions.find(clientRequestId); subIt != _subscriptions.end()) { + req_id = subIt->second.reqId;; + _subscriptions.erase(subIt); + }; + } + sendCmd(cmd.topic, cmd.command, req_id, cmd.data); + } + + DirectoryLightClient& nameserverClient() { + return _nameserver; + }; + +private: + void sendCmd(const URI<> &uri, mdp::Command commandType, std::size_t req_id, IoBuffer data = {}) const { + const bool isSet = commandType == mdp::Command::Set; + zmq::MessageFrame cmdType{ std::string{ static_cast(commandType) } }; + cmdType.send(_control_socket_send, ZMQ_SNDMORE).assertSuccess(); + zmq::MessageFrame reqId{ std::to_string(req_id) }; + reqId.send(_control_socket_send, ZMQ_SNDMORE).assertSuccess(); + zmq::MessageFrame endpoint{ std::string(uri.str()) }; + endpoint.send(_control_socket_send, isSet ? ZMQ_SNDMORE : 0).assertSuccess(); + if (isSet) { + zmq::MessageFrame dataframe{ std::move(data) }; + dataframe.send(_control_socket_send, 0).assertSuccess(); + } + } + + void handleRequests() { + zmq::MessageFrame cmd; + zmq::MessageFrame reqId; + zmq::MessageFrame endpoint; + while (cmd.receive(_control_socket_recv, ZMQ_DONTWAIT).isValid()) { + if (!reqId.receive(_control_socket_recv, ZMQ_DONTWAIT).isValid()) { + throw std::logic_error("invalid request received: failure receiving message"); + } + if (!endpoint.receive(_control_socket_recv, ZMQ_DONTWAIT).isValid()) { + throw std::logic_error("invalid request received: invalid message contents"); + } + URI uri{ std::string(endpoint.data()) }; + const auto &client = getClient(uri); + if (cmd.data().size() != 1) { + throw std::logic_error("invalid request received: wrong number of frames"); + } else if (cmd.data()[0] == static_cast(mdp::Command::Get)) { + client->get(uri, reqId.data()); + } else if (cmd.data()[0] == static_cast(mdp::Command::Set)) { + zmq::MessageFrame data; + if (!data.receive(_control_socket_recv, ZMQ_DONTWAIT).isValid()) { + throw std::logic_error("missing set data"); + } + client->set(uri, reqId.data(), std::span(data.data().data(), data.data().size())); + } else if (cmd.data()[0] == static_cast(mdp::Command::Subscribe)) { + client->subscribe(uri, reqId.data()); + } else if (cmd.data()[0] == static_cast(mdp::Command::Unsubscribe)) { + client->unsubscribe(uri, reqId.data()); + } else { + throw std::logic_error("invalid request received"); // messages always consist of 2 frames + } + } + } + + void poll(const std::stop_token &stoken) { + auto nextHousekeeping = std::chrono::system_clock::now(); + zmq::invoke(zmq_connect, _control_socket_recv, "inproc://mdclientControlSocket").assertSuccess(); + while (!stoken.stop_requested() && zmq::invoke(zmq_poll, _pollitems.data(), static_cast(_pollitems.size()), 200)) { + if (auto now = std::chrono::system_clock::now(); nextHousekeeping < now) { + nextHousekeeping = housekeeping(now); + // expire old subscriptions/requests/connections + } + handleRequests(); + for (const auto &client: _clients | std::views::values) { + mdp::Message receivedEvent; + while (client->receive(receivedEvent)) { + if (auto now = std::chrono::system_clock::now(); nextHousekeeping < now) { // perform housekeeping duties periodically + nextHousekeeping = housekeeping(now); + } + if (receivedEvent.command == mdp::Command::Invalid) { // Protocol internal messages, which require further receives but no publishing + continue; + } + std::string clientRequestId{reinterpret_cast(receivedEvent.clientRequestID.data()), receivedEvent.clientRequestID.size()}; + if (receivedEvent.command == mdp::Command::Notify && _subscriptions.contains(clientRequestId)) { + _subscriptions.at(clientRequestId).callback(receivedEvent); // callback + } + if (receivedEvent.command == mdp::Command::Final && _requests.contains(receivedEvent.id)) { + _requests.at(receivedEvent.id).callback(receivedEvent); // callback + _requests.erase(receivedEvent.id); + } + } + } + } + } + + timePoint housekeeping(const timePoint now) const { + timePoint next = now + _timeout; + for (const auto &client: _clients | std::views::values) { + next = std::min(next, client->housekeeping(now)); + } + return next; + } +}; +} // namespace opencmw::client::cmwlight +#endif // OPENCMW_CPP_CMWLIGHTCLIENT_HPP diff --git a/src/client/include/DirectoryLightClient.hpp b/src/client/include/DirectoryLightClient.hpp new file mode 100644 index 00000000..5e6020dc --- /dev/null +++ b/src/client/include/DirectoryLightClient.hpp @@ -0,0 +1,190 @@ +#ifndef OPENCMW_CPP_DIRECTORYLIGHTCLIENT_HPP +#define OPENCMW_CPP_DIRECTORYLIGHTCLIENT_HPP + +#include +#include +#include +#include + +#include + +#include + +#include + +namespace opencmw::client::cmwlight { + struct NameserverReplyLocation { + std::string domain; + std::string endpoint; + }; + struct NameserverReplyServer { + std::string name; + NameserverReplyLocation location; + }; + + struct NameserverReplyResource { + std::string name; + NameserverReplyServer server; + }; + + struct NameserverReply { + std::vector resources; + }; +} +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::NameserverReplyLocation, domain, endpoint) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::NameserverReplyServer, name, location) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::NameserverReplyResource, name, server) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::NameserverReply, resources) + +/* + * Implements the CMW Directory Server lookup + * + * TODO: + * - allow more fields than just the address to be returned + * - how to communicate nameserver errors to the client + * - timeouts for queries and TTL for cache + * +* minimal API usage example +* curl ${CMW_NAMESERVER}/api/v1/devices/search --json '{ "proxyPreferred" : true, "domains" : [ ], "directServers" : [ ], "redirects" : { }, "names" : [ "GSITemplateDevice" ]}' +* {"resources":[{"name":"GSITemplateDevice","server":{"name":"GSITemplate_DU.vmla017","location":{"domain":"RDA3","endpoint":"9#Address:#string#19#tcp:%2F%2Fvmla017:36725#ApplicationId:#string#124#app=GSITemplate_DU;uid=root;host=vmla017;pid=398;os=Linux%2D6%2E6%2E111%2Drt31%2Dyocto%2Dpreempt%2Drt;osArch=64bit;appArch=64bit;lang=C%2B%2B;#Language:#string#3#C%2B%2B#Name:#string#22#GSITemplate_DU%2Evmla017#Pid:#int#398#ProcessName:#string#14#GSITemplate_DU#StartTime:#long#1771235547903#UserName:#string#4#root#Version:#string#5#5%2E1%2E1"}}}]} +*/ + +namespace opencmw::client::cmwlight { +class DirectoryLightClient { + std::map> cache; + std::mutex mutex; + std::deque pendingLookups; + std::string nameserver; + +public: + explicit DirectoryLightClient(std::string _nameserver) : nameserver(std::move(_nameserver)) {} + + std::optional lookup(const std::string_view name, const bool useCache = true) { + { + std::lock_guard lock{ mutex }; + if (useCache) { + if (const auto &res = cache.find(name); res != cache.end()) { + if (!res->second.empty()) { + return res->second; + } else { + return {}; // request was already sent + } + } + } + cache[std::string{name}] = ""; + } + triggerRequest(name); + return {}; + } + + void addStaticLookup(const std::string & deviceName, const std::string & address) { + std::lock_guard lock{ mutex }; + cache[deviceName] = address; + }; + + static std::optional parseEndpoint(std::string_view endpoint, const std::string &classname) { + using namespace std::literals; + auto urlDecode = [](const std::string &str) { + std::string ret; + ret.reserve(str.length()); + const std::size_t len = str.length(); + for (std::size_t i = 0; i < len; i++) { + if (str[i] != '%') { + if (str[i] == '+') { + ret += ' '; + } else { + ret += str[i]; + } + } else if (i + 2 < len) { + auto toHex = [](char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + throw std::runtime_error("Invalid hexadecimal number"); + }; + const char ch = static_cast('\x10' * toHex(str.at(i + 1)) + toHex(str.at(i + 2))); + ret += ch; + i = i + 2; + } + } + return ret; + }; + auto tokens = std::views::split(endpoint, "#"sv); + if (tokens.empty()) { + return {}; + } + std::string fieldCountString = { tokens.front().data(), tokens.front().size() }; + char *end = to_address(fieldCountString.end()); + std::size_t fieldCount = std::strtoull(fieldCountString.data(), &end, 10); + + auto range = std::views::drop(tokens, 1); + auto iterator = range.begin(); + std::size_t n = 0; + while (n < fieldCount) { + std::string_view fieldNameView{ &(*iterator).front(), (*iterator).size() }; + std::string fieldname{ fieldNameView.substr(0, fieldNameView.size() - 1) }; + ++iterator; + if (std::string type{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; type == "string") { + ++iterator; + std::string sizeString{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; + auto parsed = std::to_address(sizeString.end()); + std::size_t size = std::strtoull(sizeString.data(), &parsed, 10); + ++iterator; + std::string string{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; + auto value = urlDecode(string); + if (fieldname == "Address") { + return value;; + } + } else if (type == "int") { + ++iterator; + std::string sizeString{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; + int number = std::atoi(sizeString.data()); + } else if (type == "long") { + ++iterator; + std::string sizeString{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; + auto parsed = std::to_address(sizeString.end()); + long number = std::strtol(sizeString.data(), &parsed, 10); + } else { + FAIL(std::format("unknown type: {}, field: {}, endpoint: {}", type, fieldname, endpoint)); + } + ++iterator; + ++n; + } + return {}; + } + + static std::optional> parseNameserverReply(const std::string &reply) { + IoBuffer buffer{reply.data(), reply.size()}; + NameserverReply replyObj; + auto res = opencmw::deserialise(buffer, replyObj); + for (const auto &[name, server] : replyObj.resources) { + if (server.location.domain != "RDA3") { + continue; + } + if (auto address = parseEndpoint(server.location.endpoint, name)) { + return std::optional(std::make_pair(name, address.value())); + } + } + return {}; + } + + void triggerRequest(std::string_view name) { + using namespace std::literals; + std::string requestData = std::format(R"""({{ "proxyPreferred" : true, "domains" : [ ], "directServers" : [ ], "redirects" : {{ }}, "names" : [ "{}" ]}})""", name); + const auto uri = URI<>::factory(URI(std::string(nameserver))).path("/api/v1/devices/search").build(); + std::string deviceName{name}; + cpr::PostCallback([this, deviceName](cpr::Response response) -> void { + if (response.status_code == 200) { + if (std::optional> result = parseNameserverReply(response.text); result) { + if (deviceName != result->first) { + throw std::runtime_error("unexpected device name in reply"); + } + std::lock_guard lock{ mutex }; + cache[std::string{result->first}] = result->second; + } + } + }, cpr::Url{uri.str()}, cpr::Body{requestData}, cpr::Header{{"Content-Type", "application/json"}}); + } +}; +} +#endif // OPENCMW_CPP_DIRECTORYLIGHTCLIENT_HPP diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt index ea4c7727..c4ef0cff 100644 --- a/src/client/test/CMakeLists.txt +++ b/src/client/test/CMakeLists.txt @@ -65,6 +65,16 @@ if(NOT EMSCRIPTEN) # TEST_PREFIX to whatever you want, or use different for different binaries catch_discover_tests(client_tests # TEST_PREFIX "unittests." REPORTER xml OUTPUT_DIR . OUTPUT_PREFIX "unittests." OUTPUT_SUFFIX .xml) catch_discover_tests(clientPublisher_tests) + + add_executable(CmwLightTest catch_main.cpp CmwLightTest.cpp) + target_link_libraries( + CmwLightTest + PUBLIC opencmw_project_warnings + opencmw_project_options + Catch2::Catch2 + client) + target_include_directories(CmwLightTest PRIVATE ${CMAKE_SOURCE_DIR}) + catch_discover_tests(CmwLightTest) endif() add_executable(rest_client_only_tests RestClientOnly_tests.cpp) diff --git a/src/client/test/CmwLightTest.cpp b/src/client/test/CmwLightTest.cpp new file mode 100644 index 00000000..fb2ce598 --- /dev/null +++ b/src/client/test/CmwLightTest.cpp @@ -0,0 +1,282 @@ +#include "MockServer.hpp" +#include "RestClientNative.hpp" +#include +#include +#include +#include +#include + +/** + * Most tests in this suite require a running Test Fesa Server which is registered to a Nameserver. + * Additionally, the following Environment Variables have to be set: + * - CMW_NAMESERVER=http://host:port - URL to the Nameserver to query + * - CMW_DEVICE_ADDRESS=tcp://host:port - URL where an instance of the GSITemplateDevice FESA class is running + * This could be done via nameserver lookup, but has to be set if the device needs ssh tunnelling with ssh -L local_port:device:port + * + * FESA Test Device: + * - vmla017 + * - DEV nameserver + * - Device: GSITemplateDevice + * - servername: GSITemplate_DU.vmla017 + * - Properties: + * - Acquisition: + * - Version + * - ModuleStatus + * - Status + * - Acquisition (mux) + * - Voltage: generiert jede 2 Sekunden Zufallswerte fuer Beam Process 1 und 2 + * - Setting: + * - Power + * - Init + * - Reset + * - Setting (mux) + * - Voltage: bestimmt die Acquisition-werte + * + * TODO: + * - test SET requests + */ +TEST_CASE("CmwNameserver", "[Client]") { + using namespace std::literals; + std::string nameserverExample = R"""(rda3://9#Address:#string#18#tcp:%2F%2Fdal025:16134#ApplicationId:#string#114#app=DigitizerDU2;uid=root;host=dal025;pid=16912;os=Linux%2D3%2E10%2E101%2Drt111%2Dscu03;osArch=64bit;appArch=64bit;lang=C%2B%2B;#Language:#string#3#C%2B%2B#Name:#string#19#DigitizerDU2%2Edal025#Pid:#int#16912#ProcessName:#string#12#DigitizerDU2#StartTime:#long#1699343695922#UserName:#string#4#root#Version:#string#5#3%2E1%2E0)"""; + std::string nameserverExample2 = R"""(rda3://9#Address:#string#18#tcp:%2F%2Ffel0053:3717#ApplicationId:#string#115#app=DigitizerDU2;uid=root;host=fel0053;pid=31447;os=Linux%2D3%2E10%2E101%2Drt111%2Dscu03;osArch=64bit;appArch=64bit;lang=C%2B%2B;#Language:#string#3#C%2B%2B#Name:#string#20#DigitizerDU2%2Efel0053#Pid:#int#31447#ProcessName:#string#12#DigitizerDU2#StartTime:#long#1701529074225#UserName:#string#4#root#Version:#string#5#3%2E1%2E0)"""; + std::string nameserverReply = R"""({"resources":[{"name":"GECD001","server":{"name":"DigitizerDU2.dal018","location":{"domain":"RDA3","endpoint":"9#Address:#string#17#tcp:%2F%2Fdal018:6397#ApplicationId:#string#113#app=DigitizerDU2;uid=root;host=dal018;pid=1977;os=Linux%2D3%2E10%2E101%2Drt111%2Dscu03;osArch=64bit;appArch=64bit;lang=C%2B%2B;#Language:#string#3#C%2B%2B#Name:#string#19#DigitizerDU2%2Edal018#Pid:#int#1977#ProcessName:#string#12#DigitizerDU2#StartTime:#long#1769161872191#UserName:#string#4#root#Version:#string#5#3%2E1%2E0"}}}]})"""; + + SECTION("ParseNameserverReply") { + auto result = opencmw::client::cmwlight::DirectoryLightClient::parseNameserverReply(nameserverReply); + REQUIRE(result.has_value()); + REQUIRE(result->second == R"""(tcp://dal018:6397)"""); + } + + SECTION("Query rda3 directory server/nameserver") { + auto env_nameserver = std::getenv("CMW_NAMESERVER"); + if (env_nameserver == nullptr) { + std::println("skipping BasicCmwLight example test as it relies on the availability of network infrastructure."); + return; // skip test + } + std::string nameserver{ env_nameserver }; + const opencmw::zmq::Context zctx{}; + opencmw::client::cmwlight::DirectoryLightClient nameserverClient{nameserver}; + std::optional result = nameserverClient.lookup("GSITemplateDevice"); + REQUIRE(!result.has_value()); // check that the device is originally not in the nameserver's cache + while (!(result = nameserverClient.lookup("GSITemplateDevice")).has_value()) { + std::this_thread::sleep_for(1ms); + } + REQUIRE(result.value().starts_with("tcp://vmla")); + // check that later lookups immediately return the cached result + result = nameserverClient.lookup("GSITemplateDevice"); + REQUIRE(result.value().starts_with("tcp://vmla")); + }; +} + +// small utility function that prints the content of a string in the classic hexedit way with address, hexadecimal and ascii representations +static std::string hexview(const std::string_view value, std::size_t bytesPerLine = 4) { + std::string result; + result.reserve(value.size() * 4); + std::string alpha; // temporarily store the ascii representation + alpha.reserve(8 * bytesPerLine); + std::size_t i = 0; + for (auto c : value) { + if (i % (bytesPerLine * 8) == 0) { + result.append(std::format("{0:#08x} - {0:04} | ", i)); // print address in hex and decimal + } + result.append(std::format("{:02x} ", c)); + alpha.append(std::format("{}", std::isprint(c) ? c : '.')); + if ((i + 1) % 8 == 0) { + result.append(" "); + alpha.append(" "); + } + if ((i + 1) % (bytesPerLine * 8) == 0) { + result.append(std::format(" {}\n", alpha)); + alpha.clear(); + } + i++; + } + result.append(std::format("{:{}} {}\n", "", 3 * (9 * bytesPerLine - alpha.size()), alpha)); + return result; +} + +static bool waitFor(const std::atomic &counter, const int expectedValue, const auto timeout) { + const auto start = std::chrono::system_clock::now(); + while (counter.load() < expectedValue && std::chrono::system_clock::now() - start < timeout) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + return counter.load() == expectedValue; +} +static bool waitFor(const std::atomic &counter, const int expectedValue) { + using namespace std::literals; + return waitFor(counter, expectedValue, 1000s); +} + +TEST_CASE("CmwLightClientGet", "[Client]") { + using namespace opencmw; + using namespace std::literals; + const std::string DEVICE_NAME = "GSITemplateDevice"; + const std::string STATUS_PROPERTY = "/GSITemplateDevice/Status"; + const std::string ACQUISITION_PROPERTY = "/GSITemplateDevice/Acquisition"; + const std::string SELECTOR = "FAIR.SELECTOR.C=3:S=1:P=1:T=300"; + + char * env_nameserver = std::getenv("CMW_NAMESERVER"); + if (env_nameserver == nullptr) { + std::println("skipping BasicCmwLight example test as it relies on the availability of network infrastructure. Define CMW_NAMESERVER environment variable to run this test."); + return; // skip test + } + std::string nameserver{ env_nameserver }; + const zmq::Context zctx{}; + std::vector> clients; + auto cmwlightClient = std::make_unique(zctx, nameserver, 100ms, "testclient"); + std::string deviceHost{}; // "tcp://vmla017:36725" }; + if (char * env_cmw_host = std::getenv("CMW_DEVICE_HOST"); env_cmw_host != nullptr) { + deviceHost = std::string{env_cmw_host}; + cmwlightClient->nameserverClient().addStaticLookup(DEVICE_NAME, std::string{env_cmw_host}); + } else { + while (!deviceHost.empty()) { + deviceHost = cmwlightClient->nameserverClient().lookup(DEVICE_NAME).value_or(""); + } + } + clients.emplace_back(std::move(cmwlightClient)); + client::ClientContext clientContext{ std::move(clients) }; + // send some requests + { + auto endpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(STATUS_PROPERTY).build(); + std::atomic getReceived{ 0 }; + clientContext.get(endpoint, [&getReceived](const mdp::Message &message) { + if (!message.error.empty()) { + FAIL("get should have succeeded"); + } else { + ++getReceived; + } + }); + REQUIRE(waitFor(getReceived, 1)); + } + { + auto endpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(ACQUISITION_PROPERTY).addQueryParameter("ctx", SELECTOR).build(); + std::atomic getReceived{ 0 }; + clientContext.get(endpoint, [&getReceived](const mdp::Message &message) { + if (!message.error.empty()) { + FAIL("get should have succeeded"); + } else { + ++getReceived; + } + }); + REQUIRE(waitFor(getReceived, 1)); + } + { + auto endpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path("/GSITemplateDevice/Setting").build(); + std::atomic getError{ 0 }; + clientContext.get(endpoint, [&getError](const mdp::Message &message) { + if (!message.error.empty()) { + REQUIRE(message.error.contains("Access point 'GSITemplateDevice/Setting' needs a selector")); + ++getError; + } else { + FAIL("get should have failed"); + } + }); + REQUIRE(waitFor(getError, 1)); + } + { // SET Request + + } + { // Unreachable device server + + } + { // non-existent device property + + } +} + +TEST_CASE("CmwLightClientSubscribe", "[Client]") { + using namespace opencmw; + using namespace std::literals; + const std::string DEVICE_NAME = "GSITemplateDevice"; + const std::string STATUS_PROPERTY = "/GSITemplateDevice/Status"; + const std::string ACQUISITION_PROPERTY = "/GSITemplateDevice/Acquisition"; + const std::string SELECTOR = "FAIR.SELECTOR.P=1"; + + char * env_nameserver = std::getenv("CMW_NAMESERVER"); + if (env_nameserver == nullptr) { + std::println("skipping BasicCmwLight example test as it relies on the availability of network infrastructure. Define CMW_NAMESERVER environment variable to run this test."); + return; // skip test + } + std::string nameserver{ env_nameserver }; + const zmq::Context zctx{}; + std::vector> clients; + auto cmwlightClient = std::make_unique(zctx, nameserver, 100ms, "testclient"); + std::string deviceHost{}; // "tcp://vmla017:36725" }; + if (char * env_cmw_host = std::getenv("CMW_DEVICE_HOST"); env_cmw_host != nullptr) { + deviceHost = std::string{env_cmw_host}; + cmwlightClient->nameserverClient().addStaticLookup(DEVICE_NAME, std::string{env_cmw_host}); + } else { + while (!deviceHost.empty()) { + deviceHost = cmwlightClient->nameserverClient().lookup(DEVICE_NAME).value_or(""); + } + } + clients.emplace_back(std::move(cmwlightClient)); + client::ClientContext clientContext{ std::move(clients) }; + // setup subscription + { // Subscribe non-multiplexed + std::atomic subscriptionUpdatesReceived{ 0 }; + auto subscriptionEndpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(STATUS_PROPERTY).build(); + clientContext.subscribe(subscriptionEndpoint, [&subscriptionUpdatesReceived](const mdp::Message &message) { + if (!message.error.empty()) { + FAIL("subscription should not notify exceptions"); + } else { + IoBuffer buffer(message.data); + majordomo::Empty empty{}; + auto deserialiserInfo = deserialise(buffer, empty); // deserialising into empty struct to get field information + //std::println("Deserialised subscription reply:\n fields: {}\n fieldTypes: {}", deserialiserInfo.additionalFields | std::views::keys, deserialiserInfo.additionalFields | std::views::values); + REQUIRE(deserialiserInfo.additionalFields.size() == 13); + ++subscriptionUpdatesReceived; + } + }); + REQUIRE(waitFor(subscriptionUpdatesReceived, 2, 5s)); // property gets notified every 2 seconds + // Check that subscription updates stop after unsubscribing + clientContext.unsubscribe(subscriptionEndpoint); + std::this_thread::sleep_for(200ms); // get a few subscription updates + int subscriptionUpdatesAfterUnsubscribe = subscriptionUpdatesReceived; + std::this_thread::sleep_for(5s); // get a few subscription updates + REQUIRE(subscriptionUpdatesReceived == subscriptionUpdatesAfterUnsubscribe); + } + { // error case: subscribe to multiplexed without selector + std::atomic subscriptionUpdateError{ 0 }; + auto subscriptionEndpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(ACQUISITION_PROPERTY).build(); + clientContext.subscribe(subscriptionEndpoint, [&subscriptionUpdateError](const mdp::Message &message) { + if (!message.error.empty()) { + ++subscriptionUpdateError; + REQUIRE(message.error.contains("Access point 'GSITemplateDevice/Acquisition' needs a selector")); + } else { + FAIL("invalid subscription should not notify updates"); + } + }); + REQUIRE(waitFor(subscriptionUpdateError, 1, 5s)); // property gets notified every 2 seconds + // TODO: does this need unsubscribe or is the error enough? does it do auto retry? + } + { // subscribe to muliplexed property + std::atomic subscriptionUpdatesReceived{ 0 }; + auto subscriptionEndpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(ACQUISITION_PROPERTY).addQueryParameter("ctx", SELECTOR).build(); + clientContext.subscribe(subscriptionEndpoint, [&subscriptionUpdatesReceived](const mdp::Message &message) { + if (!message.error.empty()) { + FAIL("subscription should not notify exceptions"); + } else { + IoBuffer buffer(message.data); + majordomo::Empty empty{}; + auto deserialiserInfo = deserialise(buffer, empty); // deserialising into empty struct to get field information + //std::println("Deserialised subscription reply:\n fields: {}\n fieldTypes: {}", deserialiserInfo.additionalFields | std::views::keys, deserialiserInfo.additionalFields | std::views::values); + REQUIRE(deserialiserInfo.additionalFields.size() == 12); + ++subscriptionUpdatesReceived; + } + }); + REQUIRE(waitFor(subscriptionUpdatesReceived, 2, 5s)); // property gets notified every 2 seconds + // Check that subscription updates stop after unsubscribing + clientContext.unsubscribe(subscriptionEndpoint); + std::this_thread::sleep_for(200ms); // get a few subscription updates + int subscriptionUpdatesAfterUnsubscribe = subscriptionUpdatesReceived; + std::this_thread::sleep_for(5s); // get a few subscription updates + REQUIRE(subscriptionUpdatesReceived == subscriptionUpdatesAfterUnsubscribe); + } + // SECTION("ErrorCase unreachable device server") {} + // SECTION("ErrorCase missing selector for multiplexed Property") {} + + // test filters? + // filters2String = "acquisitionModeFilter=int:0&channelNameFilter=GS11MU2:Voltage_1@10Hz"; + // subscribe("r1", new URI("rda3", null, '/' + DEVICE + '/' + PROPERTY, "ctx=" + SELECTOR + "&" + filtersString, null), null); +} diff --git a/src/core/include/SpinWait.hpp b/src/core/include/SpinWait.hpp index df0bd136..1ab2aa66 100644 --- a/src/core/include/SpinWait.hpp +++ b/src/core/include/SpinWait.hpp @@ -1,6 +1,7 @@ #ifndef SPIN_WAIT_HPP #define SPIN_WAIT_HPP +#include #include #include #include @@ -67,12 +68,12 @@ class SpinWait { void reset() noexcept { _count = 0; } template - requires std::is_nothrow_invocable_r_v + requires std::is_nothrow_invocable_r_v bool spinUntil(const T &condition) const { return spinUntil(condition, -1); } template - requires std::is_nothrow_invocable_r_v + requires std::is_nothrow_invocable_r_v bool spinUntil(const T &condition, std::int64_t millisecondsTimeout) const { if (millisecondsTimeout < -1) { @@ -111,8 +112,8 @@ class AtomicMutex { SPIN_WAIT _spin_wait; public: - AtomicMutex() = default; - AtomicMutex(const AtomicMutex &) = delete; + AtomicMutex() = default; + AtomicMutex(const AtomicMutex &) = delete; AtomicMutex &operator=(const AtomicMutex &) = delete; // diff --git a/src/core/include/Topic.hpp b/src/core/include/Topic.hpp index 6a851cb0..3fc228fa 100644 --- a/src/core/include/Topic.hpp +++ b/src/core/include/Topic.hpp @@ -103,11 +103,13 @@ struct Topic { return opencmw::URI::factory().path(_service).setQuery({ _params.begin(), _params.end() }).build(); } - std::string toZmqTopic() const { + std::string toZmqTopic(const bool delimitWithPound = true) const { using namespace std::string_literals; std::string zmqTopic = _service; if (_params.empty()) { - zmqTopic += "#"s; + if (delimitWithPound) { + zmqTopic += "#"s; + } return zmqTopic; } zmqTopic += "?"s; @@ -123,7 +125,9 @@ struct Topic { } isFirst = false; } - zmqTopic += "#"s; + if (delimitWithPound) { + zmqTopic += "#"s; + } return zmqTopic; } diff --git a/src/serialiser/include/IoSerialiser.hpp b/src/serialiser/include/IoSerialiser.hpp index 91f0e64a..d8bc6d90 100644 --- a/src/serialiser/include/IoSerialiser.hpp +++ b/src/serialiser/include/IoSerialiser.hpp @@ -147,10 +147,18 @@ struct IoSerialiser { } }; +template string> +constexpr const char *sanitizeFieldName() { + if constexpr (N > 2 && string.data[0] == 'x' && string.data[1] == '_') { + return string.c_str() + 2; + } + return string.c_str(); +} + template OPENCMW_FORCEINLINE int32_t findMemberIndex(const std::string_view &fieldName) noexcept { static constexpr ConstExprMap().members.size> m{ refl::util::map_to_array>(refl::reflect().members, [](auto field, auto index) { - return std::pair(field.name.c_str(), index); + return std::pair(sanitizeFieldName(), index); }) }; return m.at(fieldName, -1); } @@ -200,7 +208,7 @@ constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value, F using UnwrappedMemberType = std::remove_reference_t; if constexpr (isReflectableClass()) { // nested data-structure const auto subfields = getNumberOfNonNullSubfields(getAnnotatedMember(unwrapPointer(fieldValue))); - FieldDescription auto field = newFieldHeader(buffer, member.name.c_str(), parent.hierarchyDepth + 1, FWD(fieldValue), subfields); + FieldDescription auto field = newFieldHeader(buffer, sanitizeFieldName(), parent.hierarchyDepth + 1, FWD(fieldValue), subfields); const std::size_t posSizePositionStart = FieldHeaderWriter::template put(buffer, field, START_MARKER_INST); const std::size_t posStartDataStart = buffer.size(); serialise(buffer, getAnnotatedMember(unwrapPointer(fieldValue)), field); // do not inspect annotation itself @@ -208,7 +216,7 @@ constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value, F updateSize(buffer, posSizePositionStart, posStartDataStart); return; } else { // field is a (possibly annotated) primitive type - FieldDescription auto field = newFieldHeader(buffer, member.name.c_str(), parent.hierarchyDepth + 1, fieldValue, 0); + FieldDescription auto field = newFieldHeader(buffer, sanitizeFieldName(), parent.hierarchyDepth + 1, fieldValue, 0); FieldHeaderWriter::template put(buffer, field, fieldValue); } } @@ -220,7 +228,7 @@ template constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value) { putHeaderInfo(buffer); const auto subfields = detail::getNumberOfNonNullSubfields(value); - auto field = detail::newFieldHeader(buffer, refl::reflect(value).name.c_str(), 0, value, subfields); + auto field = detail::newFieldHeader(buffer, sanitizeFieldName>().name.size, refl::reflect>().name>(), 0, value, subfields); const std::size_t posSizePositionStart = FieldHeaderWriter::template put(buffer, field, START_MARKER_INST); const std::size_t posStartDataStart = buffer.size(); detail::serialise(buffer, value, field); @@ -387,7 +395,7 @@ constexpr void deserialise(IoBuffer &buffer, ReflectableClass auto &value, Deser if constexpr (isReflectableClass()) { buffer.set_position(field.headerStart); // reset buffer position for the nested deserialiser to read again field.hierarchyDepth++; - field.fieldName = member.name.c_str(); // N.B. needed since member.name is referring to compile-time const string + field.fieldName = sanitizeFieldName(); // N.B. needed since member.name is referring to compile-time const string deserialise(buffer, unwrapPointerCreateIfAbsent(member(value)), info, field); field.hierarchyDepth--; field.subfields = previousSubFields - 1; diff --git a/src/serialiser/include/IoSerialiserCmwLight.hpp b/src/serialiser/include/IoSerialiserCmwLight.hpp index ddd8286a..7f46da6f 100644 --- a/src/serialiser/include/IoSerialiserCmwLight.hpp +++ b/src/serialiser/include/IoSerialiserCmwLight.hpp @@ -5,8 +5,8 @@ #include #pragma clang diagnostic push -#pragma ide diagnostic ignored "cppcoreguidelines-avoid-magic-numbers" -#pragma ide diagnostic ignored "cppcoreguidelines-avoid-c-arrays" +#pragma ide diagnostic ignored "cppcoreguidelines-avoid-magic-numbers" +#pragma ide diagnostic ignored "cppcoreguidelines-avoid-c-arrays" namespace opencmw { @@ -18,7 +18,7 @@ namespace cmwlight { void skipString(IoBuffer &buffer); template -inline void skipArray(IoBuffer &buffer) { +void skipRawArray(IoBuffer &buffer) { if constexpr (is_stringlike) { for (auto i = buffer.get(); i > 0; i--) { skipString(buffer); @@ -28,24 +28,30 @@ inline void skipArray(IoBuffer &buffer) { } } -inline void skipString(IoBuffer &buffer) { skipArray(buffer); } +template +void skipArray(IoBuffer &buffer) { + buffer.skip(2 * static_cast(sizeof(int32_t))); + skipRawArray(buffer); +} + +inline void skipString(IoBuffer &buffer) { skipRawArray(buffer); } template -inline void skipMultiArray(IoBuffer &buffer) { +void skipMultiArray(IoBuffer &buffer) { buffer.skip(buffer.get() * static_cast(sizeof(int32_t))); // skip size header - skipArray(buffer); // skip elements + skipRawArray(buffer); // skip elements } template -inline int getTypeId() { +int getTypeId() { return IoSerialiser::getDataTypeId(); } template -inline int getTypeIdVector() { +int getTypeIdVector() { return IoSerialiser>::getDataTypeId(); } template -inline int getTypeIdMultiArray() { +int getTypeIdMultiArray() { return IoSerialiser>::getDataTypeId(); } @@ -53,12 +59,12 @@ inline int getTypeIdMultiArray() { template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFF; } // default value + static constexpr uint8_t getDataTypeId() { return 0xFF; } // default value }; template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v) { return 0; } else if constexpr (std::is_same_v) { return 1; } @@ -70,6 +76,7 @@ struct IoSerialiser { else if constexpr (std::is_same_v) { return 201; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const T &value) noexcept { @@ -83,7 +90,7 @@ struct IoSerialiser { template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 7; } + static constexpr uint8_t getDataTypeId() { return 7; } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) noexcept { buffer.put(value); @@ -96,9 +103,9 @@ struct IoSerialiser { template struct IoSerialiser { - using MemberType = typename T::value_type; + using MemberType = T::value_type; - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v) { return 9; } else if constexpr (std::is_same_v) { return 10; } @@ -111,20 +118,25 @@ struct IoSerialiser { else if constexpr (opencmw::is_stringlike ) { return 16; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const T &value) noexcept { + buffer.put(1); + buffer.put(static_cast(value.size())); buffer.put(value); } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, T &value) noexcept { - buffer.getArray(value); + const int nx = buffer.get(); + const int ny = buffer.get(); + buffer.getArray(value, nx * ny); } }; template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v && T::n_dims_ == 1) { return cmwlight::getTypeIdVector(); } else if constexpr (std::is_same_v && T::n_dims_ == 1) { return cmwlight::getTypeIdVector(); } @@ -155,6 +167,7 @@ struct IoSerialiser { else if constexpr (opencmw::is_stringlike ) { return 32; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const T &value) noexcept { @@ -169,7 +182,7 @@ struct IoSerialiser { constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, T &value) noexcept { const std::array dimWire = buffer.getArray(); for (auto i = 0U; i < T::n_dims_; i++) { - value.dimensions()[i] = static_cast(dimWire[i]); + value.dimensions()[i] = static_cast(dimWire[i]); } value.element_count() = value.dimensions()[T::n_dims_ - 1]; value.stride(T::n_dims_ - 1) = 1; @@ -185,7 +198,7 @@ struct IoSerialiser { template struct IoSerialiser> { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v) { return 9; } else if constexpr (std::is_same_v) { return 10; } @@ -198,6 +211,7 @@ struct IoSerialiser> { else if constexpr (opencmw::is_stringlike ) { return 16; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const std::set &value) noexcept { @@ -232,22 +246,24 @@ struct IoSerialiser> { } } }; + template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFC; } + static constexpr uint8_t getDataTypeId() { return 0x08; } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const START_MARKER &/*value*/) noexcept { buffer.put(static_cast(field.subfields)); } - constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*field*/, const START_MARKER &) { - // do not do anything, as the start marker is of size zero and only the type byte is important + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto &field, const START_MARKER &) { + field.subfields = static_cast(buffer.get()); + field.dataStartPosition = buffer.position(); } }; template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFE; } + static constexpr uint8_t getDataTypeId() { return 0xFE; } static void serialise(IoBuffer &/*buffer*/, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -260,7 +276,7 @@ struct IoSerialiser { template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFD; } + static constexpr uint8_t getDataTypeId() { return 0xFD; } static void serialise(IoBuffer &/*buffer*/, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -281,6 +297,7 @@ struct IoSerialiser { else if (typeId == getTypeId()) { buffer.skip(sizeof(char )); } else if (typeId == getTypeId()) { skipString(buffer); } // arrays + else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } @@ -290,6 +307,7 @@ struct IoSerialiser { else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } // 2D and Multi array + else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } @@ -318,7 +336,7 @@ struct FieldHeaderWriter { if (field.hierarchyDepth != 0) { buffer.put(field.fieldName); // full field name with zero termination } - if constexpr (!is_same_v) { // do not put startMarker type id into buffer + if (!is_same_v || field.hierarchyDepth != 0) { // do not put startMarker type id into buffer buffer.put(IoSerialiser::getDataTypeId()); // data type ID } IoSerialiser::serialise(buffer, field, getAnnotatedMember(unwrapPointer(data))); @@ -329,26 +347,30 @@ struct FieldHeaderWriter { template<> struct FieldHeaderReader { template - inline static void get(IoBuffer &buffer, DeserialiserInfo & /*info*/, FieldDescription auto &field) { + static void get(IoBuffer &buffer, DeserialiserInfo & /*info*/, FieldDescription auto &field) { field.headerStart = buffer.position(); field.dataEndPosition = std::numeric_limits::max(); - field.modifier = ExternalModifier::UNKNOWN; + field.modifier = UNKNOWN; if (field.subfields == 0) { + if (field.hierarchyDepth != 0 && field.intDataType == IoSerialiser::getDataTypeId() && buffer.get() != 0) { + throw ProtocolException("logic error, parent serialiser claims no data but header differs"); + } field.intDataType = IoSerialiser::getDataTypeId(); field.hierarchyDepth--; field.dataStartPosition = buffer.position(); return; } if (field.subfields == -1) { - if (field.hierarchyDepth != 0) { // do not read field description for root element - field.fieldName = buffer.get(); // full field name + if (field.hierarchyDepth != 0) { // do not read field description for root element + field.fieldName = buffer.get(); // full field name + field.intDataType = buffer.get(); // data type ID + } else { + field.intDataType = IoSerialiser::getDataTypeId(); } - field.intDataType = IoSerialiser::getDataTypeId(); - field.subfields = static_cast(buffer.get()); } else { field.fieldName = buffer.get(); // full field name field.intDataType = buffer.get(); // data type ID - field.subfields--; // decrease the number of remaining fields in the structure... todo: adapt strategy for nested fields (has to somewhere store subfields) + field.subfields--; // decrease the number of remaining fields in the structure... } field.dataStartPosition = buffer.position(); } @@ -364,6 +386,99 @@ inline DeserialiserInfo checkHeaderInfo(IoBuffer &buffer, Deserialiser return info; } +/** + * The serialiser for std::variant is a bit special, as it contains a runtime determined type, while in general the IoSerialiser assumes that the + * type can be deduced statically from the type via the getDataTypeId function. + * Therefore for now only serialisation is implemented and the even there we have to retroactively overwrite the field header's type ID from within the + * serialise function. + */ +template +struct IoSerialiser> { + static constexpr uint8_t getDataTypeId() { + return IoSerialiser::getDataTypeId(); // this is just a stand-in and will be overwritten with the actual type id in the serialise function + } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const std::variant &value) noexcept { + std::visit([&buffer, &field](T &val) { + using StrippedT = std::remove_cvref_t; + // overwrite field header with actual type + buffer.resize(buffer.size() - sizeof(uint8_t)); + buffer.put(IoSerialiser::getDataTypeId()); + // serialise the contained value + IoSerialiser::serialise(buffer, field, val); + }, + value); + } + + constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*parent*/, std::variant & /*value*/) noexcept { + throw ProtocolException("Deserialisation of variant types not currently supported"); + } +}; + +template +struct IoSerialiser> { + static constexpr uint8_t getDataTypeId() { + return IoSerialiser::getDataTypeId(); + } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &parentField, const std::map &value) noexcept { + buffer.put(static_cast(value.size())); + for (auto &[key, val] : value) { + if constexpr (isReflectableClass()) { // nested data-structure + const auto subfields = value.size(); + FieldDescription auto field = newFieldHeader(buffer, parentField.hierarchyDepth + 1, FWD(val), subfields); + [[maybe_unused]] const std::size_t posSizePositionStart = FieldHeaderWriter::put(buffer, field, val); + [[maybe_unused]] const std::size_t posStartDataStart = buffer.size(); + return; + } else { // field is a (possibly annotated) primitive type + FieldDescription auto field = detail::newFieldHeader(buffer, key.c_str(), parentField.hierarchyDepth + 1, val, 0); + FieldHeaderWriter::put(buffer, field, val); + } + } + } + + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &parent, std::map &value) noexcept { + DeserialiserInfo info; + constexpr ProtocolCheck check = ProtocolCheck::IGNORE; + using protocol = CmwLight; + auto field = detail::newFieldHeader(buffer, "", parent.hierarchyDepth, ValueType{}, parent.subfields); + while (buffer.position() < buffer.size()) { + auto previousSubFields = field.subfields; + FieldHeaderReader::get(buffer, info, field); + buffer.set_position(field.dataStartPosition); // skip to data start + + if (field.intDataType == IoSerialiser::getDataTypeId()) { // reached end of sub-structure + try { + IoSerialiser::deserialise(buffer, field, END_MARKER_INST); + } catch (...) { + if (detail::handleDeserialisationErrorAndSkipToNextField(buffer, field, info, "IoSerialiser<{}, END_MARKER>::deserialise(buffer, {}::{}, END_MARKER_INST): position {} vs. size {} -- exception: {}", + protocol::protocolName(), parent.fieldName, field.fieldName, buffer.position(), buffer.size(), what())) { + continue; + } + } + return; // step down to previous hierarchy depth + } + + const auto [fieldValue, _] = value.insert({ std::string{ field.fieldName }, ValueType{} }); + if constexpr (isReflectableClass()) { + field.intDataType = IoSerialiser::getDataTypeId(); + buffer.set_position(field.headerStart); // reset buffer position for the nested deserialiser to read again + field.hierarchyDepth++; + deserialise(buffer, fieldValue->second, info, field); + field.hierarchyDepth--; + field.subfields = previousSubFields - 1; + } else { + constexpr int requestedType = IoSerialiser::getDataTypeId(); + if (requestedType != field.intDataType) { // mismatching data-type + detail::moveToFieldEndBufferPosition(buffer, field); + detail::handleDeserialisationError(info, "mismatched field type for map field {} - requested type: {} (typeID: {}) got: {}", field.fieldName, typeName, requestedType, field.intDataType); + return; + } + IoSerialiser::deserialise(buffer, field, fieldValue->second); + } + } + } +}; } // namespace opencmw #pragma clang diagnostic pop diff --git a/src/serialiser/include/IoSerialiserJson.hpp b/src/serialiser/include/IoSerialiserJson.hpp index b546c51c..a99220b4 100644 --- a/src/serialiser/include/IoSerialiserJson.hpp +++ b/src/serialiser/include/IoSerialiserJson.hpp @@ -15,10 +15,10 @@ struct Json : Protocol<"Json"> {}; // as specified in https://www.json.org/json- namespace json { -constexpr inline bool isNumberChar(uint8_t c) { return (c >= '+' && c <= '9' && c != '/' && c != ',') || c == 'e' || c == 'E'; } -constexpr inline bool isWhitespace(uint8_t c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } -constexpr inline bool isHexNumberChar(int8_t c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } -constexpr inline bool isHexNumber(const char *start, const size_t size) noexcept { +constexpr bool isNumberChar(uint8_t c) { return (c >= '+' && c <= '9' && c != '/' && c != ',') || c == 'e' || c == 'E'; } +constexpr bool isWhitespace(uint8_t c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } +constexpr bool isHexNumberChar(int8_t c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } +constexpr bool isHexNumber(const char *start, const size_t size) noexcept { for (size_t i = 0; i < size; i++) { if (!isHexNumberChar(start[i])) { return false; @@ -28,7 +28,7 @@ constexpr inline bool isHexNumber(const char *start, const size_t size) noexcept } template -inline void writeString(IoBuffer &buffer, const T &string) { + void writeString(IoBuffer &buffer, const T &string) { buffer.put('"'); for (const char c : string) { switch (c) { @@ -135,19 +135,19 @@ inline std::string_view readKey(IoBuffer &buffer) { return buffer.asString(start, static_cast(i - start)); } -inline constexpr void consumeWhitespace(IoBuffer &buffer) { + constexpr void consumeWhitespace(IoBuffer &buffer) { while (buffer.position() < buffer.size() && isWhitespace(buffer.at(buffer.position()))) { buffer.skip(1); } } template -inline constexpr void assignArray(std::vector &value, const std::vector &result) { + constexpr void assignArray(std::vector &value, const std::vector &result) { value = result; } template -inline constexpr void assignArray(std::array &value, const std::vector &result) { + constexpr void assignArray(std::array &value, const std::vector &result) { if (N != result.size()) { throw ProtocolException("vector -> array size mismatch: source<{}>[{}]={} vs. destination std::array", typeName, result.size(), result, typeName, N); } @@ -156,9 +156,9 @@ inline constexpr void assignArray(std::array &value, const std::vector } } -inline constexpr void skipValue(IoBuffer &buffer); + constexpr void skipValue(IoBuffer &buffer); -inline void skipField(IoBuffer &buffer) { +inline void skipField(IoBuffer &buffer) { consumeWhitespace(buffer); std::ignore = readString(buffer); consumeWhitespace(buffer); @@ -170,7 +170,7 @@ inline void skipField(IoBuffer &buffer) { consumeWhitespace(buffer); } -inline constexpr void skipObject(IoBuffer &buffer) { + constexpr void skipObject(IoBuffer &buffer) { if (buffer.get() != '{') { throw ProtocolException("error: expected {"); } @@ -187,13 +187,13 @@ inline constexpr void skipObject(IoBuffer &buffer) { consumeWhitespace(buffer); } -inline constexpr void skipNumber(IoBuffer &buffer) { + constexpr void skipNumber(IoBuffer &buffer) { while (isNumberChar(buffer.get())) { } buffer.skip(-1); } -inline constexpr void skipArray(IoBuffer &buffer) { + constexpr void skipArray(IoBuffer &buffer) { if (buffer.get() != '[') { throw ProtocolException("error: expected ["); } @@ -211,13 +211,13 @@ inline constexpr void skipArray(IoBuffer &buffer) { consumeWhitespace(buffer); } -inline constexpr void skipValue(IoBuffer &buffer) { + constexpr void skipValue(IoBuffer &buffer) { consumeWhitespace(buffer); const auto firstChar = buffer.at(buffer.position()); if (firstChar == '{') { - json::skipObject(buffer); + skipObject(buffer); } else if (firstChar == '[') { - json::skipArray(buffer); + skipArray(buffer); } else if (firstChar == '"') { std::ignore = readString(buffer); } else if (buffer.size() - buffer.position() > 5 && buffer.asString(buffer.position(), 5) == "false") { @@ -227,17 +227,17 @@ inline constexpr void skipValue(IoBuffer &buffer) { } else if (buffer.size() - buffer.position() > 4 && buffer.asString(buffer.position(), 4) == "null") { buffer.skip(4); } else { // skip number - json::skipNumber(buffer); + skipNumber(buffer); } } } // namespace json template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 1; } - constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { + static constexpr uint8_t getDataTypeId() { return 1; } + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { using namespace std::string_view_literals; - buffer.put("}"sv); + buffer.put("}"sv); } constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*field*/, const END_MARKER &) { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -246,10 +246,10 @@ struct IoSerialiser { template<> struct IoSerialiser { // catch all template - inline static constexpr uint8_t getDataTypeId() { return 2; } - constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const START_MARKER &/*value*/) noexcept { + static constexpr uint8_t getDataTypeId() { return 2; } + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const START_MARKER &/*value*/) noexcept { using namespace std::string_view_literals; - buffer.put("{"sv); + buffer.put("{"sv); } constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*field*/, const START_MARKER & /*value*/) { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -258,8 +258,8 @@ struct IoSerialiser { // catch all template template<> struct IoSerialiser { // because json does not explicitly provide the datatype, all types except nested classes provide data type OTHER - inline static constexpr uint8_t getDataTypeId() { return 0; } - constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const OTHER &) { + static constexpr uint8_t getDataTypeId() { return 0; } + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const OTHER &) { json::skipValue(buffer); } }; @@ -269,7 +269,7 @@ struct FieldHeaderWriter { template constexpr std::size_t static put(IoBuffer &buffer, FieldDescription auto &&field, const DataType &data) { using namespace std::string_view_literals; - constexpr auto WITHOUT = opencmw::IoBuffer::MetaInfo::WITHOUT; + constexpr auto WITHOUT = IoBuffer::MetaInfo::WITHOUT; if constexpr (std::is_same_v) { if (field.fieldName.size() > 0 && field.hierarchyDepth > 0) { buffer.put("\""sv); @@ -280,7 +280,7 @@ struct FieldHeaderWriter { return 0; } if constexpr (std::is_same_v) { - if (buffer.template at(buffer.size() - 2) == ',') { + if (buffer.at(buffer.size() - 2) == ',') { // proceeded by value, remove trailing comma buffer.resize(buffer.size() - 2); } @@ -322,7 +322,7 @@ struct FieldHeaderReader { if (buffer.at(buffer.position()) == '}') { // end marker buffer.skip(1); result.intDataType = IoSerialiser::getDataTypeId(); - result.dataStartPosition = buffer.position() + 1; + result.dataStartPosition = buffer.position(); result.dataEndPosition = std::numeric_limits::max(); // not defined for non-skippable data // set rest of fields return; @@ -355,10 +355,10 @@ struct FieldHeaderReader { template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const bool &value) noexcept { using namespace std::string_view_literals; - buffer.put(value ? "true"sv : "false"sv); + buffer.put(value ? "true"sv : "false"sv); } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, bool &value) { using namespace std::string_view_literals; @@ -379,7 +379,7 @@ struct IoSerialiser { template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) { buffer.reserve_spare(30); // just reserve some spare capacity and expect that all numbers are shorter const auto start = buffer.size(); @@ -432,7 +432,7 @@ struct IoSerialiser { template struct IoSerialiser { // catch all template - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) noexcept { json::writeString(buffer, value); } @@ -444,9 +444,10 @@ struct IoSerialiser { // catch all template template struct IoSerialiser { // todo: arrays of objects - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } - inline constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &values) noexcept { - using MemberType = typename T::value_type; + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &values) noexcept { + using MemberType = T::value_type; buffer.put('['); bool first = true; for (auto value : values) { @@ -460,7 +461,7 @@ struct IoSerialiser { buffer.put(']'); } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &field, T &value) { - using MemberType = typename T::value_type; + using MemberType = T::value_type; if (buffer.get() != '[') { throw ProtocolException("expected ["); } @@ -473,7 +474,7 @@ struct IoSerialiser { IoSerialiser::deserialise(buffer, field, entry); result.push_back(entry); json::consumeWhitespace(buffer); - const auto next = buffer.template get(); + const auto next = buffer.get(); if (next == ']') { break; } @@ -489,35 +490,35 @@ struct IoSerialiser { template struct IoSerialiser { - using V = typename T::value_type; - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER + using V = T::value_type; + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &value) noexcept { using namespace std::string_view_literals; - buffer.put("{\n"sv); + buffer.put("{\n"sv); std::array dims; for (uint32_t i = 0U; i < T::n_dims_; i++) { dims[i] = static_cast(value.dimensions()[i]); } FieldDescriptionShort memberField{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; memberField.fieldName = "dims"sv; - FieldHeaderWriter::template put(buffer, memberField, dims); + FieldHeaderWriter::put(buffer, memberField, dims); memberField.fieldName = "values"sv; - FieldHeaderWriter::template put(buffer, memberField, value.elements()); + FieldHeaderWriter::put(buffer, memberField, value.elements()); buffer.resize(buffer.size() - 2); // remove trailing comma - buffer.put("\n}\n"sv); + buffer.put("\n}\n"sv); } static void deserialise(IoBuffer &buffer, FieldDescription auto const &field, T &value) { using namespace std::string_view_literals; DeserialiserInfo info{}; - FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = ExternalModifier::UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; - FieldHeaderReader::template get(buffer, info, fieldHeader); + FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; + FieldHeaderReader::get(buffer, info, fieldHeader); if (fieldHeader.intDataType != IoSerialiser>::getDataTypeId() || fieldHeader.fieldName != "dims"sv) { throw ProtocolException("expected multi-array dimensions"); } std::array dimWire; IoSerialiser>::deserialise(buffer, fieldHeader, dimWire); for (auto i = 0U; i < T::n_dims_; i++) { - value.dimensions()[i] = static_cast(dimWire[i]); + value.dimensions()[i] = static_cast(dimWire[i]); } value.element_count() = value.dimensions()[T::n_dims_ - 1]; value.stride(T::n_dims_ - 1) = 1; @@ -527,12 +528,12 @@ struct IoSerialiser { value.stride(i - 1) = value.stride(i) * value.dimensions()[i]; value.offset(i - 1) = 0; } - FieldHeaderReader::template get(buffer, info, fieldHeader); + FieldHeaderReader::get(buffer, info, fieldHeader); if (fieldHeader.intDataType != IoSerialiser>::getDataTypeId() || fieldHeader.fieldName != "values"sv) { throw ProtocolException("expected multi-array values"); } IoSerialiser>::deserialise(buffer, fieldHeader, value.elements()); - FieldHeaderReader::template get(buffer, info, fieldHeader); + FieldHeaderReader::get(buffer, info, fieldHeader); if (fieldHeader.intDataType != IoSerialiser::getDataTypeId()) { throw ProtocolException("expected end marker"); } @@ -542,8 +543,9 @@ struct IoSerialiser { template struct IoSerialiser> { // todo: arrays of objects - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } - inline constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const std::set &values) noexcept { + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const std::set &values) noexcept { buffer.put('['); bool first = true; for (auto &value : values) { @@ -569,7 +571,7 @@ struct IoSerialiser> { IoSerialiser::deserialise(buffer, field, entry); value.insert(entry); json::consumeWhitespace(buffer); - const auto next = buffer.template get(); + const auto next = buffer.get(); if (next == ']') { break; } @@ -583,30 +585,42 @@ struct IoSerialiser> { }; template requires(is_stringlike) struct IoSerialiser { - using K = typename T::key_type; - using V = typename T::mapped_type; - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER + using K = T::key_type; + using V = T::mapped_type; + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &value) { using namespace std::string_view_literals; - buffer.put("{\n"sv); + buffer.put("{\n"sv); FieldDescriptionShort memberField{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; for (auto &entry : value) { memberField.fieldName = entry.first; - FieldHeaderWriter::template put(buffer, memberField, entry.second); + FieldHeaderWriter::put(buffer, memberField, entry.second); } buffer.resize(buffer.size() - 2); // remove trailing comma - buffer.put("\n}\n"sv); + buffer.put("\n}\n"sv); } static void deserialise(IoBuffer &buffer, FieldDescription auto const &field, T &value) { using namespace std::string_view_literals; DeserialiserInfo info{}; - FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = ExternalModifier::UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; - for (FieldHeaderReader::template get(buffer, info, fieldHeader); fieldHeader.intDataType != IoSerialiser::getDataTypeId(); FieldHeaderReader::get(buffer, info, fieldHeader)) { + FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; + for (FieldHeaderReader::get(buffer, info, fieldHeader); fieldHeader.intDataType != IoSerialiser::getDataTypeId(); FieldHeaderReader::get(buffer, info, fieldHeader)) { auto fieldValue = value[std::string(fieldHeader.fieldName)]; IoSerialiser::deserialise(buffer, fieldHeader, fieldValue); } } }; + +template +struct IoSerialiser { + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) noexcept { + opencmw::serialise(buffer, value); + } + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, T &value) { + opencmw::deserialise(buffer, value); // TODO: refactor API to correctly propagate ProtocolCheck and Deserialiser Info + } +}; + } // namespace opencmw #endif // OPENCMW_JSONSERIALISER_H diff --git a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp index 4c670ad5..e83c5f25 100644 --- a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp +++ b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp @@ -4,10 +4,10 @@ #include #include +#include #include #include -#include #include #include @@ -33,12 +33,71 @@ struct SimpleTestData { opencmw::MultiArray d{ { 1, 2, 3, 4, 5, 6 }, { 2, 3 } }; std::unique_ptr e = nullptr; std::set f{ "one", "two", "three" }; - bool operator==(const ioserialiser_cmwlight_test::SimpleTestData &other) const { // deep comparison function - return a == other.a && ab == other.ab && abc == other.abc && b == other.b && c == other.c && cd == other.cd && d == other.d && ((!e && !other.e) || *e == *(other.e)) && f == other.f; + std::map g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } }; + bool operator==(const SimpleTestData &o) const { // deep comparison function + return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; + } +}; +struct SimpleTestDataMapNested { + int g1; + int g2; + int g3; + + bool operator==(const SimpleTestDataMapNested &o) const = default; + bool operator==(const std::map &o) const { + return g1 == o.at("g1") && g2 == o.at("g2") && g3 == o.at("g3"); + } +}; +struct SimpleTestDataMapAsNested { + int a = 1337; + float ab = 13.37f; + double abc = 42.23; + std::string b = "hello"; + std::array c{ 3, 2, 1 }; + std::vector cd{ 2.3, 3.4, 4.5, 5.6 }; + std::vector ce{ "hello", "world" }; + opencmw::MultiArray d{ { 1, 2, 3, 4, 5, 6 }, { 2, 3 } }; + std::unique_ptr e = nullptr; + std::set f{ "one", "two", "three" }; + SimpleTestDataMapNested g{ 1, 2, 3 }; + bool operator==(const SimpleTestDataMapAsNested &o) const { // deep comparison function + return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; + } + bool operator==(const SimpleTestData &o) const { // deep comparison function + return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; } }; } // namespace ioserialiser_cmwlight_test -ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestData, a, ab, abc, b, c, cd, ce, d, e, f) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestData, a, ab, abc, b, c, cd, ce, d, e, f, g) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMapAsNested, a, ab, abc, b, c, cd, ce, d, e, f, g) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMapNested, g1, g2, g3) + +// small utility function that prints the content of a string in the classic hexedit way with address, hexadecimal and ascii representations +static std::string hexview(const std::string_view value, const std::size_t bytesPerLine = 4) { + std::string result; + result.reserve(value.size() * 4); + std::string alpha; // temporarily store the ascii representation + alpha.reserve(8 * bytesPerLine); + std::size_t i = 0; + for (auto c : value) { + if (i % (bytesPerLine * 8) == 0) { + result.append(std::format("{0:#08x} - {0:04} | ", i)); // print address in hex and decimal + } + result.append(std::format("{:02x} ", c)); + alpha.append(std::format("{}", std::isprint(c) ? c : '.')); + if ((i + 1) % 8 == 0) { + result.append(" "); + alpha.append(" "); + } + if ((i + 1) % (bytesPerLine * 8) == 0) { + result.append(std::format(" {}\n", alpha)); + alpha.clear(); + } + i++; + } + result.append(std::format("{:{}} {}\n", "", 3 * (9 * bytesPerLine - alpha.size()), alpha)); + return result; +}; TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { using namespace opencmw; @@ -48,9 +107,34 @@ TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { debug::Timer timer("IoClassSerialiser basic syntax", 30); IoBuffer buffer; + IoBuffer bufferMap; std::cout << std::format("buffer size (before): {} bytes\n", buffer.size()); - SimpleTestData data{ + SimpleTestDataMapAsNested data{ + .a = 30, + .ab = 1.2f, + .abc = 1.23, + .b = "abc", + .c = { 5, 4, 3 }, + .cd = { 2.1, 4.2 }, + .ce = { "hallo", "welt" }, + .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, + .e = std::make_unique(SimpleTestDataMapAsNested{ + .a = 40, + .ab = 2.2f, + .abc = 2.23, + .b = "abcdef", + .c = { 9, 8, 7 }, + .cd = { 3.1, 1.2 }, + .ce = { "ei", "gude" }, + .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, + .e = nullptr, + .g = { 6, 6, 6 } }), + .f = { "four", "five" }, + .g = { 4, 5, 6 } + }; + + SimpleTestData dataMap{ .a = 30, .ab = 1.2f, .abc = 1.23, @@ -68,31 +152,52 @@ TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { .cd = { 3.1, 1.2 }, .ce = { "ei", "gude" }, .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, - .e = nullptr }), - .f = { "four", "five" } + .e = nullptr, + .g = { { "g1", 6 }, { "g2", 6 }, { "g3", 6 } } }), + .f = { "four", "five" }, + .g = { { "g1", 4 }, { "g2", 5 }, { "g3", 6 } } }; // check that empty buffer cannot be deserialised buffer.put(0L); - CHECK_THROWS_AS((opencmw::deserialise(buffer, data)), ProtocolException); + // CHECK_THROWS_AS((opencmw::deserialise(buffer, data)), ProtocolException); buffer.clear(); - SimpleTestData data2; + SimpleTestDataMapAsNested data2; REQUIRE(data != data2); + SimpleTestDataMapAsNested dataMap2; + REQUIRE(dataMap != dataMap2); std::cout << "object (short): " << ClassInfoShort << data << '\n'; std::cout << std::format("object (std::format): {}\n", data); std::cout << "object (long): " << ClassInfoVerbose << data << '\n'; - opencmw::serialise(buffer, data); + opencmw::serialise(buffer, data); + opencmw::serialise(bufferMap, dataMap); std::cout << std::format("buffer size (after): {} bytes\n", buffer.size()); - buffer.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly + buffer.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly + bufferMap.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly buffer.reset(); - auto result = opencmw::deserialise(buffer, data2); - std::cout << "deserialised object (long): " << ClassInfoVerbose << data2 << '\n'; - std::cout << "deserialisation messages: " << result << std::endl; - REQUIRE(data == data2); + bufferMap.reset(); + + std::cout << "buffer contentsSubObject: \n" + << hexview(buffer.asString()) << "\n"; + std::cout << "buffer contentsMap: \n" + << hexview(bufferMap.asString()) << "\n"; + + REQUIRE(buffer.asString() == bufferMap.asString()); + + // TODO: fix this case + // auto result = opencmw::deserialise(buffer, data2); + // std::cout << "deserialised object (long): " << ClassInfoVerbose << data2 << '\n'; + // std::cout << "deserialisation messages: " << result << std::endl; + //// REQUIRE(data == data2); + + // auto result2 = opencmw::deserialise(bufferMap, dataMap2); + // std::cout << "deserialised object (long): " << ClassInfoVerbose << dataMap2 << '\n'; + // std::cout << "deserialisation messages: " << result2 << std::endl; + // REQUIRE(dataMap == dataMap2); } REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred debug::resetStats(); @@ -120,9 +225,10 @@ struct SimpleTestDataMoreFields { bool operator==(const SimpleTestDataMoreFields &) const = default; std::unique_ptr e = nullptr; std::set f{ "one", "two", "three" }; + std::map g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } }; }; } // namespace ioserialiser_cmwlight_test -ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMoreFields, a2, ab2, abc2, b2, c2, cd2, ce2, d2, e2, a, ab, abc, b, c, cd, ce, d, e, f) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMoreFields, a2, ab2, abc2, b2, c2, cd2, ce2, d2, e2, a, ab, abc, b, c, cd, ce, d, e, f, g) #pragma clang diagnostic pop TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { @@ -144,30 +250,317 @@ TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { .cd = { 2.1, 4.2 }, .ce = { "hallo", "welt" }, .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, - .f = { "four", "six" } + .f = { "four", "six" }, + .g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } } }; SimpleTestDataMoreFields data2; std::cout << std::format("object (std::format): {}\n", data); - opencmw::serialise(buffer, data); + opencmw::serialise(buffer, data); buffer.reset(); - auto result = opencmw::deserialise(buffer, data2); + auto result = opencmw::deserialise(buffer, data2); std::cout << std::format("deserialised object (std::format): {}\n", data2); std::cout << "deserialisation messages: " << result << std::endl; - REQUIRE(result.setFields["root"] == std::vector{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1 }); + REQUIRE(result.setFields["root"] == std::vector{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 }); REQUIRE(result.additionalFields.empty()); REQUIRE(result.exceptions.empty()); std::cout << "\nand now the other way round!\n\n"; buffer.clear(); - opencmw::serialise(buffer, data2); + opencmw::serialise(buffer, data2); buffer.reset(); - auto result_back = opencmw::deserialise(buffer, data); + auto result_back = opencmw::deserialise(buffer, data); std::cout << std::format("deserialised object (std::format): {}\n", data); std::cout << "deserialisation messages: " << result_back << std::endl; - REQUIRE(result_back.setFields["root"] == std::vector{ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1 }); + REQUIRE(result_back.setFields["root"] == std::vector{ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 }); REQUIRE(result_back.additionalFields.size() == 8); REQUIRE(result_back.exceptions.size() == 8); } REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred debug::resetStats(); } + +namespace ioserialiser_cmwlight_test { +struct IntegerMap { + int x_8 = 1336; // fieldname gets mapped to "8" + int foo = 42; + int bar = 45; +}; +} // namespace ioserialiser_cmwlight_test +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::IntegerMap, x_8, foo, bar) + +TEST_CASE("IoClassSerialiserCmwLight deserialise into map", "[IoClassSerialiser]") { + using namespace opencmw; + using namespace ioserialiser_cmwlight_test; + debug::resetStats(); + { + // serialise + IoBuffer buffer; + IntegerMap input{ 23, 13, 37 }; + opencmw::serialise(buffer, input); + buffer.reset(); + REQUIRE(buffer.size() == sizeof(int32_t) /* map size */ + refl::reflect(input).members.size /* map entries */ * (sizeof(int32_t) /* string lengths */ + sizeof(uint8_t) /* type */ + sizeof(int32_t) /* int */) + 2 + 4 + 4 /* strings + \0 */); + // std::cout << hexview(buffer.asString()); + + // deserialise + std::map deserialised{}; + DeserialiserInfo info; + auto field = opencmw::detail::newFieldHeader(buffer, "map", 0, deserialised, -1); + FieldHeaderReader::get(buffer, info, field); + IoSerialiser::deserialise(buffer, field, START_MARKER_INST); + IoSerialiser>::deserialise(buffer, field, deserialised); + + // check for correctness + REQUIRE(deserialised.size() == 3); + REQUIRE(deserialised["8"] == 23); + REQUIRE(deserialised["foo"] == 13); + REQUIRE(deserialised["bar"] == 37); + } + REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred + debug::resetStats(); +} + +TEST_CASE("IoClassSerialiserCmwLight deserialise variant map", "[IoClassSerialiser]") { + using namespace opencmw; + using namespace ioserialiser_cmwlight_test; + debug::resetStats(); + { + const std::string_view expected{ "\x03\x00\x00\x00" // 3 fields + "\x02\x00\x00\x00" + "a\x00" + "\x03" + "\x23\x00\x00\x00" // "a" -> int:0x23 + "\x02\x00\x00\x00" + "b\x00" + "\x06" + "\xEC\x51\xB8\x1E\x85\xEB\xF5\x3F" // "b" -> double:1.337 + "\x02\x00\x00\x00" + "c\x00" + "\x07" + "\x04\x00\x00\x00" + "foo\x00" // "c" -> "foo" + , + 45 }; + std::map> map{ { "a", 0x23 }, { "b", 1.37 }, { "c", "foo" } }; + + IoBuffer buffer; + auto field = opencmw::detail::newFieldHeader(buffer, "map", 0, map, -1); + IoSerialiser>>::serialise(buffer, field, map); + buffer.reset(); + + // std::print("expected:\n{}\ngot:\n{}\n", hexview(expected), hexview(buffer.asString())); + REQUIRE(buffer.asString() == expected); + } + REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred + debug::resetStats(); +} + +namespace opencmw::serialiser::cmwlighttests { +struct CmwLightHeaderOptions { + int64_t b; // SOURCE_ID + std::map e; + // can potentially contain more and arbitrary data + // accessors to make code more readable + int64_t &sourceId() { return b; } + std::map sessionBody; +}; +struct CmwLightHeader { + int8_t x_2; // REQ_TYPE_TAG + int64_t x_0; // ID_TAG + std::string x_1; // DEVICE_NAME + std::string f; // PROPERTY_NAME + int8_t x_7; // UPDATE_TYPE + std::string d; // SESSION_ID + std::unique_ptr x_3; + // accessors to make code more readable + int8_t &requestType() { return x_2; } + int64_t &id() { return x_0; } + std::string &device() { return x_1; } + std::string &property() { return f; } + int8_t &updateType() { return x_7; } + std::string &sessionId() { return d; } + std::unique_ptr &options() { return x_3; } +}; +struct DigitizerVersion { + std::string classVersion; + std::string deployUnitVersion; + std::string fesaVersion; + std::string gr_flowgraph_version; + std::string gr_digitizer_version; + std::string daqAPIVersion; +}; +struct Empty {}; +struct StatusProperty { + int control; + std::vector detailedStatus; + std::vector detailedStatus_labels; + std::vector detailedStatus_severity; + std::vector error_codes; + std::vector error_cycle_names; + std::vector error_messages; + std::vector error_timestamps; + bool interlock; + bool modulesReady; + bool opReady; + int powerState; + int status; +}; +} // namespace opencmw::serialiser::cmwlighttests +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::CmwLightHeaderOptions, b, e) +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::CmwLightHeader, x_2, x_0, x_1, f, x_7, d, x_3) +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::DigitizerVersion, classVersion, deployUnitVersion, fesaVersion, gr_flowgraph_version, gr_digitizer_version, daqAPIVersion) +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::StatusProperty, control, detailedStatus, detailedStatus_labels, detailedStatus_severity, error_codes, error_cycle_names, error_messages, error_timestamps, interlock, modulesReady, opReady, powerState, status) +REFL_TYPE(opencmw::serialiser::cmwlighttests::Empty) +REFL_END + +TEST_CASE("IoClassSerialiserCmwLight Deserialise rda3 data", "[IoClassSerialiser]") { + // ensure that important rda3 messages can be properly deserialized + using namespace opencmw; + debug::resetStats(); + using namespace opencmw::serialiser::cmwlighttests; + { + std::vector rda3ConnectReply = { // + 0x07, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, /**/ 0x30, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x31, /**/ 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x00, 0x02, // + 0x00, 0x00, 0x00, 0x32, 0x00, 0x01, 0x03, 0x02, /**/ 0x00, 0x00, 0x00, 0x33, 0x00, 0x08, 0x02, 0x00, // + 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x62, 0x00, /**/ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x02, 0x00, 0x00, 0x00, 0x65, 0x00, 0x08, /**/ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // + 0x37, 0x00, 0x01, 0x70, 0x02, 0x00, 0x00, 0x00, /**/ 0x64, 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x00, // + 0x02, 0x00, 0x00, 0x00, 0x66, 0x00, 0x07, 0x01, /**/ 0x00, 0x00, 0x00, 0x00 }; // + IoBuffer buffer{ rda3ConnectReply.data(), rda3ConnectReply.size() }; + CmwLightHeader deserialised; + auto result = opencmw::deserialise(buffer, deserialised); + REQUIRE(result.additionalFields.empty()); + REQUIRE(result.exceptions.empty()); + REQUIRE(result.setFields["root"sv].size() == 7); + + REQUIRE(deserialised.requestType() == 3); + REQUIRE(deserialised.id() == 0); + REQUIRE(deserialised.device().empty()); + REQUIRE(deserialised.property().empty()); + REQUIRE(deserialised.updateType() == 0x70); + REQUIRE(deserialised.sessionId().empty()); + REQUIRE(deserialised.options()->sourceId() == 0); + REQUIRE(deserialised.options()->sessionBody.empty()); + } + { + // reply req type: session confirm + std::vector data = { 0x07, 0x00, 0x00, 0x00, // .... + 0x02, 0x00, 0x00, 0x00, 0x30, 0x00, 0x04, 0x09, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, // ....0... ........ + 0x00, 0x00, 0x00, 0x31, 0x00, 0x07, 0x08, 0x00, /**/ 0x00, 0x00, 0x47, 0x53, 0x43, 0x44, 0x30, 0x30, // ...1.... ..GSCD00 + 0x32, 0x00, 0x02, 0x00, 0x00, 0x00, 0x32, 0x00, /**/ 0x01, 0x0b, 0x02, 0x00, 0x00, 0x00, 0x33, 0x00, // 2.....2. ......3. + 0x08, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x65, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, // ........ .e...... + 0x02, 0x00, 0x00, 0x00, 0x37, 0x00, 0x01, 0x00, /**/ 0x02, 0x00, 0x00, 0x00, 0x64, 0x00, 0x07, 0x25, // ....7... ....d..% + 0x01, 0x00, 0x00, 0x52, 0x65, 0x6d, 0x6f, 0x74, /**/ 0x65, 0x48, 0x6f, 0x73, 0x74, 0x49, 0x6e, 0x66, // ...Remot eHostInf + 0x6f, 0x49, 0x6d, 0x70, 0x6c, 0x5b, 0x6e, 0x61, /**/ 0x6d, 0x65, 0x3d, 0x66, 0x65, 0x73, 0x61, 0x2d, // oImpl[na me=fesa- + 0x65, 0x78, 0x70, 0x6c, 0x6f, 0x72, 0x65, 0x72, /**/ 0x2d, 0x61, 0x70, 0x70, 0x3b, 0x20, 0x75, 0x73, // explorer -app; us + 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x61, /**/ 0x6b, 0x72, 0x69, 0x6d, 0x6d, 0x3b, 0x20, 0x61, // erName=a krimm; a + 0x70, 0x70, 0x49, 0x64, 0x3d, 0x5b, 0x61, 0x70, /**/ 0x70, 0x3d, 0x66, 0x65, 0x73, 0x61, 0x2d, 0x65, // ppId=[ap p=fesa-e + 0x78, 0x70, 0x6c, 0x6f, 0x72, 0x65, 0x72, 0x2d, /**/ 0x61, 0x70, 0x70, 0x3b, 0x76, 0x65, 0x72, 0x3d, // xplorer- app;ver= + 0x31, 0x39, 0x2e, 0x30, 0x2e, 0x30, 0x3b, 0x75, /**/ 0x69, 0x64, 0x3d, 0x61, 0x6b, 0x72, 0x69, 0x6d, // 19.0.0;u id=akrim + 0x6d, 0x3b, 0x68, 0x6f, 0x73, 0x74, 0x3d, 0x53, /**/ 0x59, 0x53, 0x50, 0x43, 0x30, 0x30, 0x38, 0x3b, // m;host=S YSPC008; + 0x70, 0x69, 0x64, 0x3d, 0x31, 0x39, 0x31, 0x36, /**/ 0x31, 0x36, 0x3b, 0x5d, 0x3b, 0x20, 0x70, 0x72, // pid=1916 16;]; pr + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x3d, 0x66, 0x65, /**/ 0x73, 0x61, 0x2d, 0x65, 0x78, 0x70, 0x6c, 0x6f, // ocess=fe sa-explo + 0x72, 0x65, 0x72, 0x2d, 0x61, 0x70, 0x70, 0x3b, /**/ 0x20, 0x70, 0x69, 0x64, 0x3d, 0x31, 0x39, 0x31, // rer-app; pid=191 + 0x36, 0x31, 0x36, 0x3b, 0x20, 0x61, 0x64, 0x64, /**/ 0x72, 0x65, 0x73, 0x73, 0x3d, 0x74, 0x63, 0x70, // 616; add ress=tcp + 0x3a, 0x2f, 0x2f, 0x53, 0x59, 0x53, 0x50, 0x43, /**/ 0x30, 0x30, 0x38, 0x3a, 0x30, 0x3b, 0x20, 0x73, // ://SYSPC 008:0; s + 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, /**/ 0x3d, 0x32, 0x30, 0x32, 0x34, 0x2d, 0x30, 0x37, // tartTime =2024-07 + 0x2d, 0x30, 0x34, 0x20, 0x31, 0x31, 0x3a, 0x31, /**/ 0x31, 0x3a, 0x31, 0x32, 0x3b, 0x20, 0x63, 0x6f, // -04 11:1 1:12; co + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, /**/ 0x54, 0x69, 0x6d, 0x65, 0x3d, 0x41, 0x62, 0x6f, // nnection Time=Abo + 0x75, 0x74, 0x20, 0x61, 0x67, 0x6f, 0x3b, 0x20, /**/ 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3d, // ut ago; version= + 0x31, 0x30, 0x2e, 0x33, 0x2e, 0x30, 0x3b, 0x20, /**/ 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, // 10.3.0; language + 0x3d, 0x4a, 0x61, 0x76, 0x61, 0x5d, 0x31, 0x00, /**/ 0x02, 0x00, 0x00, 0x00, 0x66, 0x00, 0x07, 0x08, // =Java]1. ....f... + 0x00, 0x00, 0x00, 0x56, 0x65, 0x72, 0x73, 0x69, /**/ 0x6f, 0x6e, 0x00 }; //...Versi on. + IoBuffer buffer{ data.data(), data.size() }; + CmwLightHeader deserialised; + auto result = opencmw::deserialise(buffer, deserialised); + REQUIRE(result.additionalFields.empty()); + REQUIRE(result.exceptions.empty()); + REQUIRE(result.setFields["root"sv].size() == 7); + } + { + // Reply with Req Type = Reply, gets sent after get request + std::vector data = { 0x06, 0x00, // .. + 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x63, 0x6c, /**/ 0x61, 0x73, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, // ......cl assVersi + 0x6f, 0x6e, 0x00, 0x07, 0x06, 0x00, 0x00, 0x00, /**/ 0x36, 0x2e, 0x30, 0x2e, 0x30, 0x00, 0x0e, 0x00, // on...... 6.0.0... + 0x00, 0x00, 0x64, 0x61, 0x71, 0x41, 0x50, 0x49, /**/ 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, // ..daqAPI Version. + 0x07, 0x04, 0x00, 0x00, 0x00, 0x32, 0x2e, 0x30, /**/ 0x00, 0x12, 0x00, 0x00, 0x00, 0x64, 0x65, 0x70, // .....2.0 .....dep + 0x6c, 0x6f, 0x79, 0x55, 0x6e, 0x69, 0x74, 0x56, /**/ 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x07, // loyUnitV ersion.. + 0x06, 0x00, 0x00, 0x00, 0x36, 0x2e, 0x30, 0x2e, /**/ 0x30, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x66, 0x65, // ....6.0. 0.....fe + 0x73, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, /**/ 0x6e, 0x00, 0x07, 0x06, 0x00, 0x00, 0x00, 0x37, // saVersio n......7 + 0x2e, 0x33, 0x2e, 0x30, 0x00, 0x15, 0x00, 0x00, /**/ 0x00, 0x67, 0x72, 0x5f, 0x64, 0x69, 0x67, 0x69, // .3.0.... .gr_digi + 0x74, 0x69, 0x7a, 0x65, 0x72, 0x5f, 0x76, 0x65, /**/ 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x07, 0x08, // tizer_ve rsion... + 0x00, 0x00, 0x00, 0x35, 0x2e, 0x31, 0x2e, 0x34, /**/ 0x2e, 0x30, 0x00, 0x15, 0x00, 0x00, 0x00, 0x67, // ...5.1.4 .0.....g + 0x72, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x67, 0x72, /**/ 0x61, 0x70, 0x68, 0x5f, 0x76, 0x65, 0x72, 0x73, // r_flowgr aph_vers + 0x69, 0x6f, 0x6e, 0x00, 0x07, 0x08, 0x00, 0x00, /**/ 0x00, 0x35, 0x2e, 0x30, 0x2e, 0x32, 0x2e, 0x30, // ion..... .5.0.2.0 + 0x00, 0x01, 0x62, 0x03, 0x00, 0x00, 0x00, 0x02, /**/ 0x00, 0x00, 0x00, 0x35, 0x00, 0x04, 0x88, 0x39, // ..b..... ...5...9 + 0xfe, 0x41, 0x88, 0xf7, 0xde, 0x17, 0x02, 0x00, /**/ 0x00, 0x00, 0x36, 0x00, 0x04, 0x00, 0x00, 0x00, // .A...... ..6..... + 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x78, 0x00, 0x08, 0x03, 0x00, 0x00, 0x00, // ........ .x...... + 0x09, 0x00, 0x00, 0x00, 0x61, 0x63, 0x71, 0x53, /**/ 0x74, 0x61, 0x6d, 0x70, 0x00, 0x04, 0x88, 0x39, // ....acqS tamp...9 + 0xfe, 0x41, 0x88, 0xf7, 0xde, 0x17, 0x05, 0x00, /**/ 0x00, 0x00, 0x74, 0x79, 0x70, 0x65, 0x00, 0x03, // .A...... ..type.. + 0x02, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, /**/ 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, // ........ version. + 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x03} ; // ....... + IoBuffer buffer{ data.data(), data.size() }; + DigitizerVersion deserialised; + auto result = opencmw::deserialise(buffer, deserialised); + REQUIRE(result.additionalFields.empty()); + REQUIRE(result.exceptions.empty()); + REQUIRE(result.setFields["root"sv].size() == 6); + } + { + std::vector data{ // + 0x0d, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, /**/ 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x00, /**/ 0x03, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, /**/ 0x00, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, // ........ control. ........ .detaile + 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x00, /**/ 0x09, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x01, 0x16, /**/ 0x00, 0x00, 0x00, 0x64, 0x65, 0x74, 0x61, 0x69, // dStatus. ........ ........ ...detai + 0x6c, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, /**/ 0x73, 0x5f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, /**/ 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, /**/ 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x0f, 0x00, // ledStatu s_labels ........ ........ + 0x00, 0x00, 0x6d, 0x79, 0x53, 0x74, 0x61, 0x74, /**/ 0x75, 0x73, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x31, /**/ 0x00, 0x0f, 0x00, 0x00, 0x00, 0x6d, 0x79, 0x53, /**/ 0x74, 0x61, 0x74, 0x75, 0x73, 0x4c, 0x61, 0x62, // ..myStat usLabel1 .....myS tatusLab + 0x65, 0x6c, 0x32, 0x00, 0x18, 0x00, 0x00, 0x00, /**/ 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, /**/ 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x73, /**/ 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x00, // el2..... detailed Status_s everity. + 0x0c, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, /**/ 0x00, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, // ........ ........ ........ .error_c + 0x6f, 0x64, 0x65, 0x73, 0x00, 0x0c, 0x01, 0x00, /**/ 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x10, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // odes.... ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x65, 0x72, /**/ 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x79, 0x63, 0x6c, // ........ ........ ......er ror_cycl + 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x00, /**/ 0x10, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, /**/ 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /**/ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, // e_names. ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, /**/ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /**/ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, /**/ 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, /**/ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, /**/ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /**/ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, /**/ 0x00, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, /**/ 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x00, /**/ 0x10, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, // ........ .error_m essages. ........ + 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /**/ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, /**/ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, /**/ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, /**/ 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /**/ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, /**/ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /**/ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, /**/ 0x01, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, /**/ 0x00, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x74, // ........ ........ ........ .error_t + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, /**/ 0x73, 0x00, 0x0d, 0x01, 0x00, 0x00, 0x00, 0x10, /**/ 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // imestamp s....... ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, /**/ 0x00, 0x00, 0x00, 0x69, 0x6e, 0x74, 0x65, 0x72, // ........ ........ ........ ...inter + 0x6c, 0x6f, 0x63, 0x6b, 0x00, 0x00, 0x00, 0x0d, /**/ 0x00, 0x00, 0x00, 0x6d, 0x6f, 0x64, 0x75, 0x6c, /**/ 0x65, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x00, /**/ 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x6f, 0x70, // lock.... ...modul esReady. ......op + 0x52, 0x65, 0x61, 0x64, 0x79, 0x00, 0x00, 0x01, /**/ 0x0b, 0x00, 0x00, 0x00, 0x70, 0x6f, 0x77, 0x65, /**/ 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x00, 0x03, /**/ 0x01, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, // Ready... ....powe rState.. ........ + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x00, 0x03, /**/ 0x01, 0x00, 0x00, 0x00}; // status.. .... + IoBuffer buffer{ data.data(), data.size() }; + REQUIRE(buffer.size() == 748); + // deserialise once into an empty struct to verify that the fields can be correctly skipped and end up in the deserialiserInfo + Empty empty; + auto result = opencmw::deserialise(buffer, empty); + REQUIRE(result.additionalFields.size() == 13); // [root::control, root::detailedStatus, root::detailedStatus_labels, root::detailedStatus_severity, root::error_codes, root::error_cycle_names, root::error_messages, root::error_timestamps, root::interlock, root::modulesReady, root::opReady, root::powerState, root::status] [3, 9, 16, 12, 12, 16, 16, 13, 0, 0, 0, 3, 3] + REQUIRE(result.exceptions.size() == 13); // each missing field also produces an excception + REQUIRE(result.setFields["root"].empty()); + // deserialise into the correct domain object + buffer.reset(); + StatusProperty status; + auto result2 = opencmw::deserialise(buffer, status); + REQUIRE(result2.additionalFields.empty()); + REQUIRE(result2.exceptions.empty()); + REQUIRE(result2.setFields["root"sv].size() == 13); + REQUIRE(status.control == 0); + REQUIRE(status.detailedStatus == std::vector{true, true}); + REQUIRE(status.detailedStatus_labels == std::vector{"myStatusLabel1"s, "myStatusLabel2"s}); + REQUIRE(status.detailedStatus_severity == std::vector{0, 0}); + REQUIRE(status.error_codes.size() == 16); + REQUIRE(status.error_cycle_names.size() == 16); + REQUIRE(status.error_messages.size() == 16); + REQUIRE(status.error_timestamps.size() == 16); + REQUIRE_FALSE(status.interlock); + REQUIRE(status.modulesReady); + REQUIRE(status.opReady); + REQUIRE(status.powerState == 1); + REQUIRE(status.status == 1); + } + REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred + debug::resetStats(); +} diff --git a/src/serialiser/test/IoSerialiserJson_tests.cpp b/src/serialiser/test/IoSerialiserJson_tests.cpp index 4bed5887..ae2784f9 100644 --- a/src/serialiser/test/IoSerialiserJson_tests.cpp +++ b/src/serialiser/test/IoSerialiserJson_tests.cpp @@ -128,11 +128,12 @@ TEST_CASE("JsonDeserialisationMissingField", "[JsonSerialiser]") { opencmw::debug::resetStats(); { opencmw::IoBuffer buffer; - buffer.put(R"({ "float1": 2.3, "superfluousField": { "p":12 , "a":null,"x" : false, "q": [ "a", "s"], "z": [true , false ] }, "test": { "intArray" : [ 1,2, 3], "val1":13.37e2, "val2":"bar"}, "int1": 42})"sv); + buffer.put(R"({ "float1": 2.3, "superfluousField": { "p":12 , "a":null,"x" : false, "q": [ "a", "s"], "z": [true , false ] }, "test": { "intArray" : [ 1,2, 3], "val1":13.37e2, "val2":"bar"}, "int1": 42} )"sv); std::cout << "Prepared json data: " << buffer.asString() << std::endl; Simple foo; auto result = opencmw::deserialise(buffer, foo); std::print(std::cout, "deserialisation finished: {}\n", result); + REQUIRE(buffer.position() == 189UZ); REQUIRE(foo.test.get()->val1 == 1337.0); REQUIRE(foo.test.get()->val2 == "bar"); REQUIRE(foo.test.get()->intArray == std::vector{ 1, 2, 3 }); diff --git a/src/zmq/include/zmq/ZmqUtils.hpp b/src/zmq/include/zmq/ZmqUtils.hpp index 209bb369..d50571ba 100644 --- a/src/zmq/include/zmq/ZmqUtils.hpp +++ b/src/zmq/include/zmq/ZmqUtils.hpp @@ -231,6 +231,16 @@ class MessageFrame { copy); } + explicit MessageFrame(char c) { + const auto copy = new char[1] {c}; + zmq_msg_init_data( + &_message, copy, 1, + [](void * /*unused*/, void *bufOwned) { + delete static_cast(bufOwned); + }, + copy); + } + static MessageFrame fromStaticData(std::string_view buf) { MessageFrame mf; zmq_msg_init_data(