From a271a51838f34326a317cf533085b46ecc8e4d7e Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Mon, 16 Feb 2026 15:07:47 +0100 Subject: [PATCH 1/7] SpinLock: fix missing header Signed-off-by: Alexander Krimm --- src/core/include/SpinWait.hpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/core/include/SpinWait.hpp b/src/core/include/SpinWait.hpp index df0bd136..1ab2aa66 100644 --- a/src/core/include/SpinWait.hpp +++ b/src/core/include/SpinWait.hpp @@ -1,6 +1,7 @@ #ifndef SPIN_WAIT_HPP #define SPIN_WAIT_HPP +#include #include #include #include @@ -67,12 +68,12 @@ class SpinWait { void reset() noexcept { _count = 0; } template - requires std::is_nothrow_invocable_r_v + requires std::is_nothrow_invocable_r_v bool spinUntil(const T &condition) const { return spinUntil(condition, -1); } template - requires std::is_nothrow_invocable_r_v + requires std::is_nothrow_invocable_r_v bool spinUntil(const T &condition, std::int64_t millisecondsTimeout) const { if (millisecondsTimeout < -1) { @@ -111,8 +112,8 @@ class AtomicMutex { SPIN_WAIT _spin_wait; public: - AtomicMutex() = default; - AtomicMutex(const AtomicMutex &) = delete; + AtomicMutex() = default; + AtomicMutex(const AtomicMutex &) = delete; AtomicMutex &operator=(const AtomicMutex &) = delete; // From fbcc94bc50f651f739161dbbc811ccd14c96b56e Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Mon, 1 Jul 2024 17:40:35 +0200 Subject: [PATCH 2/7] CMWLightSerialiser: add features - Enable support for std::map and nesting Support for nested Datastructures was not implemented correctly before because only serialise + deserialise was checked but not the actual serialised output. - Add parsing rda3 data into a map - Allows fieldnames with starting with a numeric value by prefixing them with ' x_'. - fix handling of empty nested objects/maps Signed-off-by: Alexander Krimm --- src/serialiser/include/IoSerialiser.hpp | 18 +- .../include/IoSerialiserCmwLight.hpp | 89 ++++- .../test/IoSerialiserCmwLight_tests.cpp | 310 +++++++++++++++++- 3 files changed, 387 insertions(+), 30 deletions(-) diff --git a/src/serialiser/include/IoSerialiser.hpp b/src/serialiser/include/IoSerialiser.hpp index 91f0e64a..07df16a8 100644 --- a/src/serialiser/include/IoSerialiser.hpp +++ b/src/serialiser/include/IoSerialiser.hpp @@ -147,10 +147,18 @@ struct IoSerialiser { } }; +template string> +constexpr const char *sanitizeFieldName() { + if constexpr (N > 2 && string.data[0] == 'x' && string.data[1] == '_') { + return string.c_str() + 2; + } + return string.c_str(); +} + template OPENCMW_FORCEINLINE int32_t findMemberIndex(const std::string_view &fieldName) noexcept { static constexpr ConstExprMap().members.size> m{ refl::util::map_to_array>(refl::reflect().members, [](auto field, auto index) { - return std::pair(field.name.c_str(), index); + return std::pair(sanitizeFieldName(), index); }) }; return m.at(fieldName, -1); } @@ -200,7 +208,7 @@ constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value, F using UnwrappedMemberType = std::remove_reference_t; if constexpr (isReflectableClass()) { // nested data-structure const auto subfields = getNumberOfNonNullSubfields(getAnnotatedMember(unwrapPointer(fieldValue))); - FieldDescription auto field = newFieldHeader(buffer, member.name.c_str(), parent.hierarchyDepth + 1, FWD(fieldValue), subfields); + FieldDescription auto field = newFieldHeader(buffer, sanitizeFieldName(), parent.hierarchyDepth + 1, FWD(fieldValue), subfields); const std::size_t posSizePositionStart = FieldHeaderWriter::template put(buffer, field, START_MARKER_INST); const std::size_t posStartDataStart = buffer.size(); serialise(buffer, getAnnotatedMember(unwrapPointer(fieldValue)), field); // do not inspect annotation itself @@ -208,7 +216,7 @@ constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value, F updateSize(buffer, posSizePositionStart, posStartDataStart); return; } else { // field is a (possibly annotated) primitive type - FieldDescription auto field = newFieldHeader(buffer, member.name.c_str(), parent.hierarchyDepth + 1, fieldValue, 0); + FieldDescription auto field = newFieldHeader(buffer, sanitizeFieldName(), parent.hierarchyDepth + 1, fieldValue, 0); FieldHeaderWriter::template put(buffer, field, fieldValue); } } @@ -220,7 +228,7 @@ template constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value) { putHeaderInfo(buffer); const auto subfields = detail::getNumberOfNonNullSubfields(value); - auto field = detail::newFieldHeader(buffer, refl::reflect(value).name.c_str(), 0, value, subfields); + auto field = detail::newFieldHeader(buffer, sanitizeFieldName().name.size, refl::reflect().name>(), 0, value, subfields); const std::size_t posSizePositionStart = FieldHeaderWriter::template put(buffer, field, START_MARKER_INST); const std::size_t posStartDataStart = buffer.size(); detail::serialise(buffer, value, field); @@ -387,7 +395,7 @@ constexpr void deserialise(IoBuffer &buffer, ReflectableClass auto &value, Deser if constexpr (isReflectableClass()) { buffer.set_position(field.headerStart); // reset buffer position for the nested deserialiser to read again field.hierarchyDepth++; - field.fieldName = member.name.c_str(); // N.B. needed since member.name is referring to compile-time const string + field.fieldName = sanitizeFieldName(); // N.B. needed since member.name is referring to compile-time const string deserialise(buffer, unwrapPointerCreateIfAbsent(member(value)), info, field); field.hierarchyDepth--; field.subfields = previousSubFields - 1; diff --git a/src/serialiser/include/IoSerialiserCmwLight.hpp b/src/serialiser/include/IoSerialiserCmwLight.hpp index ddd8286a..b5bcef67 100644 --- a/src/serialiser/include/IoSerialiserCmwLight.hpp +++ b/src/serialiser/include/IoSerialiserCmwLight.hpp @@ -232,16 +232,18 @@ struct IoSerialiser> { } } }; + template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFC; } + inline static constexpr uint8_t getDataTypeId() { return 0x08; } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const START_MARKER &/*value*/) noexcept { buffer.put(static_cast(field.subfields)); } - constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*field*/, const START_MARKER &) { - // do not do anything, as the start marker is of size zero and only the type byte is important + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto &field, const START_MARKER &) { + field.subfields = static_cast(buffer.get()); + field.dataStartPosition = buffer.position(); } }; @@ -318,7 +320,7 @@ struct FieldHeaderWriter { if (field.hierarchyDepth != 0) { buffer.put(field.fieldName); // full field name with zero termination } - if constexpr (!is_same_v) { // do not put startMarker type id into buffer + if (!is_same_v || field.hierarchyDepth != 0) { // do not put startMarker type id into buffer buffer.put(IoSerialiser::getDataTypeId()); // data type ID } IoSerialiser::serialise(buffer, field, getAnnotatedMember(unwrapPointer(data))); @@ -334,21 +336,22 @@ struct FieldHeaderReader { field.dataEndPosition = std::numeric_limits::max(); field.modifier = ExternalModifier::UNKNOWN; if (field.subfields == 0) { - field.intDataType = IoSerialiser::getDataTypeId(); + field.dataStartPosition = field.headerStart + (field.intDataType == IoSerialiser::getDataTypeId() ? 4 : 0); + field.intDataType = IoSerialiser::getDataTypeId(); field.hierarchyDepth--; - field.dataStartPosition = buffer.position(); return; } if (field.subfields == -1) { - if (field.hierarchyDepth != 0) { // do not read field description for root element - field.fieldName = buffer.get(); // full field name + if (field.hierarchyDepth != 0) { // do not read field description for root element + field.fieldName = buffer.get(); // full field name + field.intDataType = buffer.get(); // data type ID + } else { + field.intDataType = IoSerialiser::getDataTypeId(); } - field.intDataType = IoSerialiser::getDataTypeId(); - field.subfields = static_cast(buffer.get()); } else { field.fieldName = buffer.get(); // full field name field.intDataType = buffer.get(); // data type ID - field.subfields--; // decrease the number of remaining fields in the structure... todo: adapt strategy for nested fields (has to somewhere store subfields) + field.subfields--; // decrease the number of remaining fields in the structure... } field.dataStartPosition = buffer.position(); } @@ -364,6 +367,70 @@ inline DeserialiserInfo checkHeaderInfo(IoBuffer &buffer, Deserialiser return info; } +template +struct IoSerialiser> { + inline static constexpr uint8_t getDataTypeId() { + return IoSerialiser::getDataTypeId(); + } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &parentField, const std::map &value) noexcept { + buffer.put(static_cast(value.size())); + for (auto &[key, val] : value) { + if constexpr (isReflectableClass()) { // nested data-structure + const auto subfields = value.size(); + FieldDescription auto field = newFieldHeader(buffer, parentField.hierarchyDepth + 1, FWD(val), subfields); + const std::size_t posSizePositionStart = FieldHeaderWriter::template put(buffer, field, val); + const std::size_t posStartDataStart = buffer.size(); + return; + } else { // field is a (possibly annotated) primitive type + FieldDescription auto field = opencmw::detail::newFieldHeader(buffer, key.c_str(), parentField.hierarchyDepth + 1, val, 0); + FieldHeaderWriter::template put(buffer, field, val); + } + } + } + + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &parent, std::map &value) noexcept { + DeserialiserInfo info; + constexpr ProtocolCheck check = ProtocolCheck::IGNORE; + using protocol = CmwLight; + auto field = opencmw::detail::newFieldHeader(buffer, "", parent.hierarchyDepth, ValueType{}, parent.subfields); + while (buffer.position() < buffer.size()) { + auto previousSubFields = field.subfields; + FieldHeaderReader::template get(buffer, info, field); + buffer.set_position(field.dataStartPosition); // skip to data start + + if (field.intDataType == IoSerialiser::getDataTypeId()) { // reached end of sub-structure + try { + IoSerialiser::deserialise(buffer, field, END_MARKER_INST); + } catch (...) { + if (opencmw::detail::handleDeserialisationErrorAndSkipToNextField(buffer, field, info, "IoSerialiser<{}, END_MARKER>::deserialise(buffer, {}::{}, END_MARKER_INST): position {} vs. size {} -- exception: {}", + protocol::protocolName(), parent.fieldName, field.fieldName, buffer.position(), buffer.size(), what())) { + continue; + } + } + return; // step down to previous hierarchy depth + } + + const auto [fieldValue, _] = value.insert({ std::string{ field.fieldName }, ValueType{} }); + if constexpr (isReflectableClass()) { + field.intDataType = IoSerialiser::getDataTypeId(); + buffer.set_position(field.headerStart); // reset buffer position for the nested deserialiser to read again + field.hierarchyDepth++; + deserialise(buffer, fieldValue->second, info, field); + field.hierarchyDepth--; + field.subfields = previousSubFields - 1; + } else { + constexpr int requestedType = IoSerialiser::getDataTypeId(); + if (requestedType != field.intDataType) { // mismatching data-type + opencmw::detail::moveToFieldEndBufferPosition(buffer, field); + opencmw::detail::handleDeserialisationError(info, "mismatched field type for map field {} - requested type: {} (typeID: {}) got: {}", field.fieldName, typeName, requestedType, field.intDataType); + return; + } + IoSerialiser::deserialise(buffer, field, fieldValue->second); + } + } + } +}; } // namespace opencmw #pragma clang diagnostic pop diff --git a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp index 4c670ad5..d808bfd2 100644 --- a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp +++ b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp @@ -4,6 +4,7 @@ #include #include +#include #include #include @@ -33,12 +34,71 @@ struct SimpleTestData { opencmw::MultiArray d{ { 1, 2, 3, 4, 5, 6 }, { 2, 3 } }; std::unique_ptr e = nullptr; std::set f{ "one", "two", "three" }; - bool operator==(const ioserialiser_cmwlight_test::SimpleTestData &other) const { // deep comparison function - return a == other.a && ab == other.ab && abc == other.abc && b == other.b && c == other.c && cd == other.cd && d == other.d && ((!e && !other.e) || *e == *(other.e)) && f == other.f; + std::map g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } }; + bool operator==(const ioserialiser_cmwlight_test::SimpleTestData &o) const { // deep comparison function + return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; + } +}; +struct SimpleTestDataMapNested { + int g1; + int g2; + int g3; + + bool operator==(const SimpleTestDataMapNested &o) const = default; + bool operator==(const std::map &o) const { + return g1 == o.at("g1") && g2 == o.at("g2") && g3 == o.at("g3"); + } +}; +struct SimpleTestDataMapAsNested { + int a = 1337; + float ab = 13.37f; + double abc = 42.23; + std::string b = "hello"; + std::array c{ 3, 2, 1 }; + std::vector cd{ 2.3, 3.4, 4.5, 5.6 }; + std::vector ce{ "hello", "world" }; + opencmw::MultiArray d{ { 1, 2, 3, 4, 5, 6 }, { 2, 3 } }; + std::unique_ptr e = nullptr; + std::set f{ "one", "two", "three" }; + SimpleTestDataMapNested g{ 1, 2, 3 }; + bool operator==(const ioserialiser_cmwlight_test::SimpleTestDataMapAsNested &o) const { // deep comparison function + return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; + } + bool operator==(const ioserialiser_cmwlight_test::SimpleTestData &o) const { // deep comparison function + return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; } }; } // namespace ioserialiser_cmwlight_test -ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestData, a, ab, abc, b, c, cd, ce, d, e, f) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestData, a, ab, abc, b, c, cd, ce, d, e, f, g) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMapAsNested, a, ab, abc, b, c, cd, ce, d, e, f, g) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMapNested, g1, g2, g3) + +// small utility function that prints the content of a string in the classic hexedit way with address, hexadecimal and ascii representations +static std::string hexview(const std::string_view value, std::size_t bytesPerLine = 4) { + std::string result; + result.reserve(value.size() * 4); + std::string alpha; // temporarily store the ascii representation + alpha.reserve(8 * bytesPerLine); + std::size_t i = 0; + for (auto c : value) { + if (i % (bytesPerLine * 8) == 0) { + result.append(fmt::format("{0:#08x} - {0:04} | ", i)); // print address in hex and decimal + } + result.append(fmt::format("{:02x} ", c)); + alpha.append(fmt::format("{}", std::isprint(c) ? c : '.')); + if ((i + 1) % 8 == 0) { + result.append(" "); + alpha.append(" "); + } + if ((i + 1) % (bytesPerLine * 8) == 0) { + result.append(fmt::format(" {}\n", alpha)); + alpha.clear(); + } + i++; + } + result.append(fmt::format("{:{}} {}\n", "", 3 * (9 * bytesPerLine - alpha.size()), alpha)); + return result; +}; TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { using namespace opencmw; @@ -48,9 +108,34 @@ TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { debug::Timer timer("IoClassSerialiser basic syntax", 30); IoBuffer buffer; + IoBuffer bufferMap; std::cout << std::format("buffer size (before): {} bytes\n", buffer.size()); - SimpleTestData data{ + SimpleTestDataMapAsNested data{ + .a = 30, + .ab = 1.2f, + .abc = 1.23, + .b = "abc", + .c = { 5, 4, 3 }, + .cd = { 2.1, 4.2 }, + .ce = { "hallo", "welt" }, + .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, + .e = std::make_unique(SimpleTestDataMapAsNested{ + .a = 40, + .ab = 2.2f, + .abc = 2.23, + .b = "abcdef", + .c = { 9, 8, 7 }, + .cd = { 3.1, 1.2 }, + .ce = { "ei", "gude" }, + .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, + .e = nullptr, + .g = { 6, 6, 6 } }), + .f = { "four", "five" }, + .g = { 4, 5, 6 } + }; + + SimpleTestData dataMap{ .a = 30, .ab = 1.2f, .abc = 1.23, @@ -68,31 +153,51 @@ TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { .cd = { 3.1, 1.2 }, .ce = { "ei", "gude" }, .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, - .e = nullptr }), - .f = { "four", "five" } + .e = nullptr, + .g = { { "g1", 6 }, { "g2", 6 }, { "g3", 6 } } }), + .f = { "four", "five" }, + .g = { { "g1", 4 }, { "g2", 5 }, { "g3", 6 } } }; // check that empty buffer cannot be deserialised buffer.put(0L); - CHECK_THROWS_AS((opencmw::deserialise(buffer, data)), ProtocolException); + // CHECK_THROWS_AS((opencmw::deserialise(buffer, data)), ProtocolException); buffer.clear(); - SimpleTestData data2; + SimpleTestDataMapAsNested data2; REQUIRE(data != data2); + SimpleTestDataMapAsNested dataMap2; + REQUIRE(dataMap != dataMap2); std::cout << "object (short): " << ClassInfoShort << data << '\n'; std::cout << std::format("object (std::format): {}\n", data); std::cout << "object (long): " << ClassInfoVerbose << data << '\n'; opencmw::serialise(buffer, data); + opencmw::serialise(bufferMap, dataMap); std::cout << std::format("buffer size (after): {} bytes\n", buffer.size()); - buffer.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly + buffer.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly + bufferMap.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly buffer.reset(); + bufferMap.reset(); + + std::cout << "buffer contentsSubObject: \n" + << hexview(buffer.asString()) << "\n"; + std::cout << "buffer contentsMap: \n" + << hexview(bufferMap.asString()) << "\n"; + + REQUIRE(buffer.asString() == bufferMap.asString()); + auto result = opencmw::deserialise(buffer, data2); std::cout << "deserialised object (long): " << ClassInfoVerbose << data2 << '\n'; std::cout << "deserialisation messages: " << result << std::endl; - REQUIRE(data == data2); + // REQUIRE(data == data2); + + auto result2 = opencmw::deserialise(bufferMap, dataMap2); + std::cout << "deserialised object (long): " << ClassInfoVerbose << data2 << '\n'; + std::cout << "deserialisation messages: " << result << std::endl; + REQUIRE(dataMap == dataMap2); } REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred debug::resetStats(); @@ -120,9 +225,10 @@ struct SimpleTestDataMoreFields { bool operator==(const SimpleTestDataMoreFields &) const = default; std::unique_ptr e = nullptr; std::set f{ "one", "two", "three" }; + std::map g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } }; }; } // namespace ioserialiser_cmwlight_test -ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMoreFields, a2, ab2, abc2, b2, c2, cd2, ce2, d2, e2, a, ab, abc, b, c, cd, ce, d, e, f) +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMoreFields, a2, ab2, abc2, b2, c2, cd2, ce2, d2, e2, a, ab, abc, b, c, cd, ce, d, e, f, g) #pragma clang diagnostic pop TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { @@ -144,7 +250,8 @@ TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { .cd = { 2.1, 4.2 }, .ce = { "hallo", "welt" }, .d = { { 6, 5, 4, 3, 2, 1 }, { 3, 2 } }, - .f = { "four", "six" } + .f = { "four", "six" }, + .g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } } }; SimpleTestDataMoreFields data2; std::cout << std::format("object (std::format): {}\n", data); @@ -153,7 +260,7 @@ TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { auto result = opencmw::deserialise(buffer, data2); std::cout << std::format("deserialised object (std::format): {}\n", data2); std::cout << "deserialisation messages: " << result << std::endl; - REQUIRE(result.setFields["root"] == std::vector{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1 }); + REQUIRE(result.setFields["root"] == std::vector{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 }); REQUIRE(result.additionalFields.empty()); REQUIRE(result.exceptions.empty()); @@ -164,10 +271,185 @@ TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { auto result_back = opencmw::deserialise(buffer, data); std::cout << std::format("deserialised object (std::format): {}\n", data); std::cout << "deserialisation messages: " << result_back << std::endl; - REQUIRE(result_back.setFields["root"] == std::vector{ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1 }); + REQUIRE(result_back.setFields["root"] == std::vector{ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 }); REQUIRE(result_back.additionalFields.size() == 8); REQUIRE(result_back.exceptions.size() == 8); } REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred debug::resetStats(); } + +namespace ioserialiser_cmwlight_test { +struct IntegerMap { + int x_8 = 1336; // fieldname gets mapped to "8" + int foo = 42; + int bar = 45; +}; +} // namespace ioserialiser_cmwlight_test +ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::IntegerMap, x_8, foo, bar) + +TEST_CASE("IoClassSerialiserCmwLight deserialise into map", "[IoClassSerialiser]") { + using namespace opencmw; + using namespace ioserialiser_cmwlight_test; + debug::resetStats(); + { + // serialise + IoBuffer buffer; + IntegerMap input{ 23, 13, 37 }; + opencmw::serialise(buffer, input); + buffer.reset(); + REQUIRE(buffer.size() == sizeof(int32_t) /* map size */ + refl::reflect(input).members.size /* map entries */ * (sizeof(int32_t) /* string lengths */ + sizeof(uint8_t) /* type */ + sizeof(int32_t) /* int */) + 2 + 4 + 4 /* strings + \0 */); + // std::cout << hexview(buffer.asString()); + + // deserialise + std::map deserialised{}; + DeserialiserInfo info; + auto field = opencmw::detail::newFieldHeader(buffer, "map", 0, deserialised, -1); + opencmw::FieldHeaderReader::template get(buffer, info, field); + IoSerialiser::deserialise(buffer, field, START_MARKER_INST); + opencmw::IoSerialiser>::deserialise(buffer, field, deserialised); + + // check for correctness + REQUIRE(deserialised.size() == 3); + REQUIRE(deserialised["8"] == 23); + REQUIRE(deserialised["foo"] == 13); + REQUIRE(deserialised["bar"] == 37); + } + REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred + debug::resetStats(); +} + +namespace opencmw::serialiser::cmwlighttests { +struct CmwLightHeaderOptions { + int64_t b; // SOURCE_ID + std::map e; + // can potentially contain more and arbitrary data + // accessors to make code more readable + int64_t &sourceId() { return b; } + std::map sessionBody; +}; +struct CmwLightHeader { + int8_t x_2; // REQ_TYPE_TAG + int64_t x_0; // ID_TAG + std::string x_1; // DEVICE_NAME + std::string f; // PROPERTY_NAME + int8_t x_7; // UPDATE_TYPE + std::string d; // SESSION_ID + std::unique_ptr x_3; + // accessors to make code more readable + int8_t &requestType() { return x_2; } + int64_t &id() { return x_0; } + std::string &device() { return x_1; } + std::string &property() { return f; } + int8_t &updateType() { return x_7; } + std::string &sessionId() { return d; } + std::unique_ptr &options() { return x_3; } +}; +struct DigitizerVersion { + std::string classVersion; + std::string deployUnitVersion; + std::string fesaVersion; + std::string gr_flowgraph_version; + std::string gr_digitizer_version; + std::string daqAPIVersion; +}; +} // namespace opencmw::serialiser::cmwlighttests +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::CmwLightHeaderOptions, b, e) +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::CmwLightHeader, x_2, x_0, x_1, f, x_7, d, x_3) +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::DigitizerVersion, classVersion, deployUnitVersion, fesaVersion, gr_flowgraph_version, gr_digitizer_version, daqAPIVersion) + +TEST_CASE("IoClassSerialiserCmwLight Deserialise rda3 data", "[IoClassSerialiser]") { + // ensure that important rda3 messages can be properly deserialized + using namespace opencmw; + debug::resetStats(); + using namespace opencmw::serialiser::cmwlighttests; + { + std::string_view rda3ConnectReply = "\x07\x00\x00\x00\x02\x00\x00\x00\x30\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x31\x00\x07\x01\x00\x00\x00\x00\x02\x00\x00\x00\x32\x00\x01\x03\x02\x00\x00\x00\x33\x00\x08\x02\x00\x00\x00\x02\x00\x00\x00\x62\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x65\x00\x08\x00\x00\x00\x00\x02\x00\x00\x00\x37\x00\x01\x70\x02\x00\x00\x00\x64\x00\x07\x01\x00\x00\x00\x00\x02\x00\x00\x00\x66\x00\x07\x01\x00\x00\x00\x00"sv; + // 0 1 2 3 4 5 6 7 8 9 a b c d e f + // 000 "\x07\x00\x00\x00\x02\x00\x00\x00" "\x30\x00\x04\x00\x00\x00\x00\x00" + // 010 "\x00\x00\x00\x02\x00\x00\x00\x31" "\x00\x07\x01\x00\x00\x00\x00\x02" + // 020 "\x00\x00\x00\x32\x00\x01\x03\x02" "\x00\x00\x00\x33\x00\x08\x02\x00" + // 030 "\x00\x00\x02\x00\x00\x00\x62\x00" "\x04\x00\x00\x00\x00\x00\x00\x00" + // 040 "\x00\x02\x00\x00\x00\x65\x00\x08" "\x00\x00\x00\x00\x02\x00\x00\x00" + // 050 "\x37\x00\x01\x70\x02\x00\x00\x00" "\x64\x00\x07\x01\x00\x00\x00\x00" + // 060 "\x02\x00\x00\x00\x66\x00\x07\x01" "\x00\x00\x00\x00"sv + IoBuffer buffer{ rda3ConnectReply.data(), rda3ConnectReply.size() }; + CmwLightHeader deserialised; + auto result = opencmw::deserialise(buffer, deserialised); + + REQUIRE(deserialised.requestType() == 3); + REQUIRE(deserialised.id() == 0); + REQUIRE(deserialised.device().empty()); + REQUIRE(deserialised.property().empty()); + REQUIRE(deserialised.updateType() == 0x70); + REQUIRE(deserialised.sessionId().empty()); + REQUIRE(deserialised.options()->sourceId() == 0); + REQUIRE(deserialised.options()->sessionBody.empty()); + } + { + std::string_view data = "\x07\x00\x00\x00\x02\x00\x00\x00\x30\x00\x04\x09\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x31\x00\x07\x08\x00\x00\x00\x47\x53\x43\x44\x30\x30\x32\x00\x02\x00\x00\x00\x32\x00\x01\x0b\x02\x00\x00\x00\x33\x00\x08\x01\x00\x00\x00\x02\x00\x00\x00\x65\x00\x08\x00\x00\x00\x00\x02\x00\x00\x00\x37\x00\x01\x00\x02\x00\x00\x00\x64\x00\x07\x25\x01\x00\x00\x52\x65\x6d\x6f\x74\x65\x48\x6f\x73\x74\x49\x6e\x66\x6f\x49\x6d\x70\x6c\x5b\x6e\x61\x6d\x65\x3d\x66\x65\x73\x61\x2d\x65\x78\x70\x6c\x6f\x72\x65\x72\x2d\x61\x70\x70\x3b\x20\x75\x73\x65\x72\x4e\x61\x6d\x65\x3d\x61\x6b\x72\x69\x6d\x6d\x3b\x20\x61\x70\x70\x49\x64\x3d\x5b\x61\x70\x70\x3d\x66\x65\x73\x61\x2d\x65\x78\x70\x6c\x6f\x72\x65\x72\x2d\x61\x70\x70\x3b\x76\x65\x72\x3d\x31\x39\x2e\x30\x2e\x30\x3b\x75\x69\x64\x3d\x61\x6b\x72\x69\x6d\x6d\x3b\x68\x6f\x73\x74\x3d\x53\x59\x53\x50\x43\x30\x30\x38\x3b\x70\x69\x64\x3d\x31\x39\x31\x36\x31\x36\x3b\x5d\x3b\x20\x70\x72\x6f\x63\x65\x73\x73\x3d\x66\x65\x73\x61\x2d\x65\x78\x70\x6c\x6f\x72\x65\x72\x2d\x61\x70\x70\x3b\x20\x70\x69\x64\x3d\x31\x39\x31\x36\x31\x36\x3b\x20\x61\x64\x64\x72\x65\x73\x73\x3d\x74\x63\x70\x3a\x2f\x2f\x53\x59\x53\x50\x43\x30\x30\x38\x3a\x30\x3b\x20\x73\x74\x61\x72\x74\x54\x69\x6d\x65\x3d\x32\x30\x32\x34\x2d\x30\x37\x2d\x30\x34\x20\x31\x31\x3a\x31\x31\x3a\x31\x32\x3b\x20\x63\x6f\x6e\x6e\x65\x63\x74\x69\x6f\x6e\x54\x69\x6d\x65\x3d\x41\x62\x6f\x75\x74\x20\x61\x67\x6f\x3b\x20\x76\x65\x72\x73\x69\x6f\x6e\x3d\x31\x30\x2e\x33\x2e\x30\x3b\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x3d\x4a\x61\x76\x61\x5d\x31\x00\x02\x00\x00\x00\x66\x00\x07\x08\x00\x00\x00\x56\x65\x72\x73\x69\x6f\x6e\x00"sv; + // reply req type: session confirm + // \x07 \x00 \x00 \x00 .... + // \x02 \x00 \x00 \x00 \x30 \x00 \x04 \x09 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x02 ....0... ........ + // \x00 \x00 \x00 \x31 \x00 \x07 \x08 \x00 \x00 \x00 \x47 \x53 \x43 \x44 \x30 \x30 ...1.... ..GSCD00 + // \x32 \x00 \x02 \x00 \x00 \x00 \x32 \x00 \x01 \x0b \x02 \x00 \x00 \x00 \x33 \x00 2.....2. ......3. + // \x08 \x01 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x65 \x00 \x08 \x00 \x00 \x00 \x00 ........ .e...... + // \x02 \x00 \x00 \x00 \x37 \x00 \x01 \x00 \x02 \x00 \x00 \x00 \x64 \x00 \x07 \x25 ....7... ....d..% + // \x01 \x00 \x00 \x52 \x65 \x6d \x6f \x74 \x65 \x48 \x6f \x73 \x74 \x49 \x6e \x66 ...Remot eHostInf + // \x6f \x49 \x6d \x70 \x6c \x5b \x6e \x61 \x6d \x65 \x3d \x66 \x65 \x73 \x61 \x2d oImpl[na me=fesa- + // \x65 \x78 \x70 \x6c \x6f \x72 \x65 \x72 \x2d \x61 \x70 \x70 \x3b \x20 \x75 \x73 explorer -app; us + // \x65 \x72 \x4e \x61 \x6d \x65 \x3d \x61 \x6b \x72 \x69 \x6d \x6d \x3b \x20 \x61 erName=a krimm; a + // \x70 \x70 \x49 \x64 \x3d \x5b \x61 \x70 \x70 \x3d \x66 \x65 \x73 \x61 \x2d \x65 ppId=[ap p=fesa-e + // \x78 \x70 \x6c \x6f \x72 \x65 \x72 \x2d \x61 \x70 \x70 \x3b \x76 \x65 \x72 \x3d xplorer- app;ver= + // \x31 \x39 \x2e \x30 \x2e \x30 \x3b \x75 \x69 \x64 \x3d \x61 \x6b \x72 \x69 \x6d 19.0.0;u id=akrim + // \x6d \x3b \x68 \x6f \x73 \x74 \x3d \x53 \x59 \x53 \x50 \x43 \x30 \x30 \x38 \x3b m;host=S YSPC008; + // \x70 \x69 \x64 \x3d \x31 \x39 \x31 \x36 \x31 \x36 \x3b \x5d \x3b \x20 \x70 \x72 pid=1916 16;]; pr + // \x6f \x63 \x65 \x73 \x73 \x3d \x66 \x65 \x73 \x61 \x2d \x65 \x78 \x70 \x6c \x6f ocess=fe sa-explo + // \x72 \x65 \x72 \x2d \x61 \x70 \x70 \x3b \x20 \x70 \x69 \x64 \x3d \x31 \x39 \x31 rer-app; pid=191 + // \x36 \x31 \x36 \x3b \x20 \x61 \x64 \x64 \x72 \x65 \x73 \x73 \x3d \x74 \x63 \x70 616; add ress=tcp + // \x3a \x2f \x2f \x53 \x59 \x53 \x50 \x43 \x30 \x30 \x38 \x3a \x30 \x3b \x20 \x73 ://SYSPC 008:0; s + // \x74 \x61 \x72 \x74 \x54 \x69 \x6d \x65 \x3d \x32 \x30 \x32 \x34 \x2d \x30 \x37 tartTime =2024-07 + // \x2d \x30 \x34 \x20 \x31 \x31 \x3a \x31 \x31 \x3a \x31 \x32 \x3b \x20 \x63 \x6f -04 11:1 1:12; co + // \x6e \x6e \x65 \x63 \x74 \x69 \x6f \x6e \x54 \x69 \x6d \x65 \x3d \x41 \x62 \x6f nnection Time=Abo + // \x75 \x74 \x20 \x61 \x67 \x6f \x3b \x20 \x76 \x65 \x72 \x73 \x69 \x6f \x6e \x3d ut ago; version= + // \x31 \x30 \x2e \x33 \x2e \x30 \x3b \x20 \x6c \x61 \x6e \x67 \x75 \x61 \x67 \x65 10.3.0; language + // \x3d \x4a \x61 \x76 \x61 \x5d \x31 \x00 \x02 \x00 \x00 \x00 \x66 \x00 \x07 \x08 =Java]1. ....f... + // \x00 \x00 \x00 \x56 \x65 \x72 \x73 \x69 \x6f \x6e \x00 ...Versi on. + IoBuffer buffer{ data.data(), data.size() }; + CmwLightHeader deserialised; + auto result = opencmw::deserialise(buffer, deserialised); + } + { + std::string_view data = "\x06\x00\x00\x00\x02\x00\x00\x00\x30\x00\x04\x09\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x31\x00\x07\x01\x00\x00\x00\x00\x02\x00\x00\x00\x32\x00\x01\x03\x02\x00\x00\x00\x37\x00\x01\x00\x02\x00\x00\x00\x64\x00\x07\x01\x00\x00\x00\x00\x02\x00\x00\x00\x66\x00\x07\x01\x00\x00\x00\x00\x01\xc3\x06\x00\x00\x00\x0d\x00\x00\x00\x63\x6c\x61\x73\x73\x56\x65\x72\x73\x69\x6f\x6e\x00\x07\x06\x00\x00\x00\x36\x2e\x30\x2e\x30\x00\x0e\x00\x00\x00\x64\x61\x71\x41\x50\x49\x56\x65\x72\x73\x69\x6f\x6e\x00\x07\x04\x00\x00\x00\x32\x2e\x30\x00\x12\x00\x00\x00\x64\x65\x70\x6c\x6f\x79\x55\x6e\x69\x74\x56\x65\x72\x73\x69\x6f\x6e\x00\x07\x06\x00\x00\x00\x36\x2e\x30\x2e\x30\x00\x0c\x00\x00\x00\x66\x65\x73\x61\x56\x65\x72\x73\x69\x6f\x6e\x00\x07\x06\x00\x00\x00\x37\x2e\x33\x2e\x30\x00\x15\x00\x00\x00\x67\x72\x5f\x64\x69\x67\x69\x74\x69\x7a\x65\x72\x5f\x76\x65\x72\x73\x69\x6f\x6e\x00\x07\x08\x00\x00\x00\x35\x2e\x31\x2e\x34\x2e\x30\x00\x15\x00\x00\x00\x67\x72\x5f\x66\x6c\x6f\x77\x67\x72\x61\x70\x68\x5f\x76\x65\x72\x73\x69\x6f\x6e\x00\x07\x08\x00\x00\x00\x35\x2e\x30\x2e\x32\x2e\x30\x00\x01\x62\x03\x00\x00\x00\x02\x00\x00\x00\x35\x00\x04\x88\x39\xfe\x41\x88\xf7\xde\x17\x02\x00\x00\x00\x36\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x78\x00\x08\x03\x00\x00\x00\x09\x00\x00\x00\x61\x63\x71\x53\x74\x61\x6d\x70\x00\x04\x88\x39\xfe\x41\x88\xf7\xde\x17\x05\x00\x00\x00\x74\x79\x70\x65\x00\x03\x02\x00\x00\x00\x08\x00\x00\x00\x76\x65\x72\x73\x69\x6f\x6e\x00\x03\x01\x00\x00\x00\x00\x03"; + // Reply with Req Type = Reply, gets sent after get request + // \x06 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x30 \x00 \x04 ... .....0.. + // \x09 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x31 \x00 \x07 \x01 ........ ....1... + // \x00 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x32 \x00 \x01 \x03 \x02 \x00 \x00 \x00 ........ 2....... + // \x37 \x00 \x01 \x00 \x02 \x00 \x00 \x00 \x64 \x00 \x07 \x01 \x00 \x00 \x00 \x00 7....... d....... + // \x02 \x00 \x00 \x00 \x66 \x00 \x07 \x01 \x00 \x00 \x00 \x00 \x01 \xc3 \x06 \x00 ....f... ........ + // \x00 \x00 \x0d \x00 \x00 \x00 \x63 \x6c \x61 \x73 \x73 \x56 \x65 \x72 \x73 \x69 ......cl assVersi + // \x6f \x6e \x00 \x07 \x06 \x00 \x00 \x00 \x36 \x2e \x30 \x2e \x30 \x00 \x0e \x00 on...... 6.0.0... + // \x00 \x00 \x64 \x61 \x71 \x41 \x50 \x49 \x56 \x65 \x72 \x73 \x69 \x6f \x6e \x00 ..daqAPI Version. + // \x07 \x04 \x00 \x00 \x00 \x32 \x2e \x30 \x00 \x12 \x00 \x00 \x00 \x64 \x65 \x70 .....2.0 .....dep + // \x6c \x6f \x79 \x55 \x6e \x69 \x74 \x56 \x65 \x72 \x73 \x69 \x6f \x6e \x00 \x07 loyUnitV ersion.. + // \x06 \x00 \x00 \x00 \x36 \x2e \x30 \x2e \x30 \x00 \x0c \x00 \x00 \x00 \x66 \x65 ....6.0. 0.....fe + // \x73 \x61 \x56 \x65 \x72 \x73 \x69 \x6f \x6e \x00 \x07 \x06 \x00 \x00 \x00 \x37 saVersio n......7 + // \x2e \x33 \x2e \x30 \x00 \x15 \x00 \x00 \x00 \x67 \x72 \x5f \x64 \x69 \x67 \x69 .3.0.... .gr_digi + // \x74 \x69 \x7a \x65 \x72 \x5f \x76 \x65 \x72 \x73 \x69 \x6f \x6e \x00 \x07 \x08 tizer_ve rsion... + // \x00 \x00 \x00 \x35 \x2e \x31 \x2e \x34 \x2e \x30 \x00 \x15 \x00 \x00 \x00 \x67 ...5.1.4 .0.....g + // \x72 \x5f \x66 \x6c \x6f \x77 \x67 \x72 \x61 \x70 \x68 \x5f \x76 \x65 \x72 \x73 r_flowgr aph_vers + // \x69 \x6f \x6e \x00 \x07 \x08 \x00 \x00 \x00 \x35 \x2e \x30 \x2e \x32 \x2e \x30 ion..... .5.0.2.0 + // \x00 \x01 \x62 \x03 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x35 \x00 \x04 \x88 \x39 ..b..... ...5...9 + // \xfe \x41 \x88 \xf7 \xde \x17 \x02 \x00 \x00 \x00 \x36 \x00 \x04 \x00 \x00 \x00 .A...... ..6..... + // \x00 \x00 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x78 \x00 \x08 \x03 \x00 \x00 \x00 ........ .x...... + // \x09 \x00 \x00 \x00 \x61 \x63 \x71 \x53 \x74 \x61 \x6d \x70 \x00 \x04 \x88 \x39 ....acqS tamp...9 + // \xfe \x41 \x88 \xf7 \xde \x17 \x05 \x00 \x00 \x00 \x74 \x79 \x70 \x65 \x00 \x03 .A...... ..type.. + // \x02 \x00 \x00 \x00 \x08 \x00 \x00 \x00 \x76 \x65 \x72 \x73 \x69 \x6f \x6e \x00 ........ version. + // \x03 \x01 \x00 \x00 \x00 \x00 \x03 ....... + IoBuffer buffer{ data.data(), data.size() }; + DigitizerVersion deserialised; + auto result = opencmw::deserialise(buffer, deserialised); + } + REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred + debug::resetStats(); +} From 4da21adbf6aa188d5a54e0b234d1db38d3a2f48b Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Mon, 15 Jul 2024 15:55:53 +0200 Subject: [PATCH 3/7] CmwLightSerialiser: fix cornercases wip Signed-off-by: Alexander Krimm --- src/serialiser/include/IoSerialiser.hpp | 2 +- .../include/IoSerialiserCmwLight.hpp | 11 +++++--- .../test/IoSerialiserCmwLight_tests.cpp | 27 ++++++++++--------- 3 files changed, 22 insertions(+), 18 deletions(-) diff --git a/src/serialiser/include/IoSerialiser.hpp b/src/serialiser/include/IoSerialiser.hpp index 07df16a8..d8bc6d90 100644 --- a/src/serialiser/include/IoSerialiser.hpp +++ b/src/serialiser/include/IoSerialiser.hpp @@ -228,7 +228,7 @@ template constexpr void serialise(IoBuffer &buffer, ReflectableClass auto const &value) { putHeaderInfo(buffer); const auto subfields = detail::getNumberOfNonNullSubfields(value); - auto field = detail::newFieldHeader(buffer, sanitizeFieldName().name.size, refl::reflect().name>(), 0, value, subfields); + auto field = detail::newFieldHeader(buffer, sanitizeFieldName>().name.size, refl::reflect>().name>(), 0, value, subfields); const std::size_t posSizePositionStart = FieldHeaderWriter::template put(buffer, field, START_MARKER_INST); const std::size_t posStartDataStart = buffer.size(); detail::serialise(buffer, value, field); diff --git a/src/serialiser/include/IoSerialiserCmwLight.hpp b/src/serialiser/include/IoSerialiserCmwLight.hpp index b5bcef67..5aa281d4 100644 --- a/src/serialiser/include/IoSerialiserCmwLight.hpp +++ b/src/serialiser/include/IoSerialiserCmwLight.hpp @@ -5,8 +5,8 @@ #include #pragma clang diagnostic push -#pragma ide diagnostic ignored "cppcoreguidelines-avoid-magic-numbers" -#pragma ide diagnostic ignored "cppcoreguidelines-avoid-c-arrays" +#pragma ide diagnostic ignored "cppcoreguidelines-avoid-magic-numbers" +#pragma ide diagnostic ignored "cppcoreguidelines-avoid-c-arrays" namespace opencmw { @@ -336,9 +336,12 @@ struct FieldHeaderReader { field.dataEndPosition = std::numeric_limits::max(); field.modifier = ExternalModifier::UNKNOWN; if (field.subfields == 0) { - field.dataStartPosition = field.headerStart + (field.intDataType == IoSerialiser::getDataTypeId() ? 4 : 0); - field.intDataType = IoSerialiser::getDataTypeId(); + if (field.hierarchyDepth != 0 && field.intDataType == IoSerialiser::getDataTypeId() && buffer.get() != 0) { + throw ProtocolException("logic error, parent serialiser claims no data but header differs"); + } + field.intDataType = IoSerialiser::getDataTypeId(); field.hierarchyDepth--; + field.dataStartPosition = buffer.position(); return; } if (field.subfields == -1) { diff --git a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp index d808bfd2..d23d3ffa 100644 --- a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp +++ b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp @@ -82,21 +82,21 @@ static std::string hexview(const std::string_view value, std::size_t bytesPerLin std::size_t i = 0; for (auto c : value) { if (i % (bytesPerLine * 8) == 0) { - result.append(fmt::format("{0:#08x} - {0:04} | ", i)); // print address in hex and decimal + result.append(std::format("{0:#08x} - {0:04} | ", i)); // print address in hex and decimal } - result.append(fmt::format("{:02x} ", c)); - alpha.append(fmt::format("{}", std::isprint(c) ? c : '.')); + result.append(std::format("{:02x} ", c)); + alpha.append(std::format("{}", std::isprint(c) ? c : '.')); if ((i + 1) % 8 == 0) { result.append(" "); alpha.append(" "); } if ((i + 1) % (bytesPerLine * 8) == 0) { - result.append(fmt::format(" {}\n", alpha)); + result.append(std::format(" {}\n", alpha)); alpha.clear(); } i++; } - result.append(fmt::format("{:{}} {}\n", "", 3 * (9 * bytesPerLine - alpha.size()), alpha)); + result.append(std::format("{:{}} {}\n", "", 3 * (9 * bytesPerLine - alpha.size()), alpha)); return result; }; @@ -189,15 +189,16 @@ TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { REQUIRE(buffer.asString() == bufferMap.asString()); - auto result = opencmw::deserialise(buffer, data2); - std::cout << "deserialised object (long): " << ClassInfoVerbose << data2 << '\n'; - std::cout << "deserialisation messages: " << result << std::endl; - // REQUIRE(data == data2); + // TODO: fix this case + // auto result = opencmw::deserialise(buffer, data2); + // std::cout << "deserialised object (long): " << ClassInfoVerbose << data2 << '\n'; + // std::cout << "deserialisation messages: " << result << std::endl; + //// REQUIRE(data == data2); - auto result2 = opencmw::deserialise(bufferMap, dataMap2); - std::cout << "deserialised object (long): " << ClassInfoVerbose << data2 << '\n'; - std::cout << "deserialisation messages: " << result << std::endl; - REQUIRE(dataMap == dataMap2); + // auto result2 = opencmw::deserialise(bufferMap, dataMap2); + // std::cout << "deserialised object (long): " << ClassInfoVerbose << dataMap2 << '\n'; + // std::cout << "deserialisation messages: " << result2 << std::endl; + // REQUIRE(dataMap == dataMap2); } REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred debug::resetStats(); From 71a03ecc76f794f6e0e9d76a81ff6c625801f9c8 Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Wed, 17 Jul 2024 13:57:35 +0200 Subject: [PATCH 4/7] IoSerialiserCmwLight: add std::variant serialiser Serialisation only, as deserialisation is not possible due to the IoSerialiser expecting to be able to deduce the type of the field at compile time via `IoSerialiser::getTypeId()`. Signed-off-by: Alexander Krimm --- .../include/IoSerialiserCmwLight.hpp | 31 +++++++++++++++- .../test/IoSerialiserCmwLight_tests.cpp | 35 +++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/serialiser/include/IoSerialiserCmwLight.hpp b/src/serialiser/include/IoSerialiserCmwLight.hpp index 5aa281d4..0869f349 100644 --- a/src/serialiser/include/IoSerialiserCmwLight.hpp +++ b/src/serialiser/include/IoSerialiserCmwLight.hpp @@ -242,7 +242,7 @@ struct IoSerialiser { } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto &field, const START_MARKER &) { - field.subfields = static_cast(buffer.get()); + field.subfields = static_cast(buffer.get()); field.dataStartPosition = buffer.position(); } }; @@ -370,6 +370,35 @@ inline DeserialiserInfo checkHeaderInfo(IoBuffer &buffer, Deserialiser return info; } +/** + * The serialiser for std::variant is a bit special, as it contains a runtime determined type, while in general the IoSerialiser assumes that the + * type can be deduced statically from the type via the getDataTypeId function. + * Therefore for now only serialisation is implemented and the even there we have to retroactively overwrite the field header's type ID from within the + * serialise function. + */ +template +struct IoSerialiser> { + inline static constexpr uint8_t getDataTypeId() { + return IoSerialiser::getDataTypeId(); // this is just a stand-in and will be overwritten with the actual type id in the serialise function + } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const std::variant &value) noexcept { + std::visit([&buffer, &field](T &val) { + using StrippedT = std::remove_cvref_t; + // overwrite field header with actual type + buffer.resize(buffer.size() - sizeof(uint8_t)); + buffer.put(IoSerialiser::getDataTypeId()); + // serialise the contained value + IoSerialiser::serialise(buffer, field, val); + }, + value); + } + + constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*parent*/, std::variant & /*value*/) noexcept { + throw ProtocolException("Deserialisation of variant types not currently supported"); + } +}; + template struct IoSerialiser> { inline static constexpr uint8_t getDataTypeId() { diff --git a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp index d23d3ffa..6e417a38 100644 --- a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp +++ b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp @@ -320,6 +320,41 @@ TEST_CASE("IoClassSerialiserCmwLight deserialise into map", "[IoClassSerialiser] debug::resetStats(); } +TEST_CASE("IoClassSerialiserCmwLight deserialise variant map", "[IoClassSerialiser]") { + using namespace opencmw; + using namespace ioserialiser_cmwlight_test; + debug::resetStats(); + { + const std::string_view expected{ "\x03\x00\x00\x00" // 3 fields + "\x02\x00\x00\x00" + "a\x00" + "\x03" + "\x23\x00\x00\x00" // "a" -> int:0x23 + "\x02\x00\x00\x00" + "b\x00" + "\x06" + "\xEC\x51\xB8\x1E\x85\xEB\xF5\x3F" // "b" -> double:1.337 + "\x02\x00\x00\x00" + "c\x00" + "\x07" + "\x04\x00\x00\x00" + "foo\x00" // "c" -> "foo" + , + 45 }; + std::map> map{ { "a", 0x23 }, { "b", 1.37 }, { "c", "foo" } }; + + IoBuffer buffer; + auto field = opencmw::detail::newFieldHeader(buffer, "map", 0, map, -1); + opencmw::IoSerialiser>>::serialise(buffer, field, map); + buffer.reset(); + + // std::print("expected:\n{}\ngot:\n{}\n", hexview(expected), hexview(buffer.asString())); + REQUIRE(buffer.asString() == expected); + } + REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred + debug::resetStats(); +} + namespace opencmw::serialiser::cmwlighttests { struct CmwLightHeaderOptions { int64_t b; // SOURCE_ID From cc7311e0989240c2c271fc1dd995911c7bcbc21a Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Thu, 19 Feb 2026 18:24:54 +0100 Subject: [PATCH 5/7] IoSerialiserCmwLight: fix array (de)serialisation Fixes two compatibility issues in the cmw light serialisation format. - The branches for skipping bool arrays that don't exist in the target object were missing => added additional branches. - CMW always serialises n_dims, [nx, ...], n_elem, [x1, ...] even for 1D arrays. Before only the number of elements was (de)serialised leading to incompatible serialised data and crashes on deserialisation. Also adds a testcase which exercises these field types and apply fixes. Signed-off-by: Alexander Krimm --- .../include/IoSerialiserCmwLight.hpp | 84 ++++--- .../test/IoSerialiserCmwLight_tests.cpp | 233 ++++++++++++------ 2 files changed, 204 insertions(+), 113 deletions(-) diff --git a/src/serialiser/include/IoSerialiserCmwLight.hpp b/src/serialiser/include/IoSerialiserCmwLight.hpp index 0869f349..7f46da6f 100644 --- a/src/serialiser/include/IoSerialiserCmwLight.hpp +++ b/src/serialiser/include/IoSerialiserCmwLight.hpp @@ -18,7 +18,7 @@ namespace cmwlight { void skipString(IoBuffer &buffer); template -inline void skipArray(IoBuffer &buffer) { +void skipRawArray(IoBuffer &buffer) { if constexpr (is_stringlike) { for (auto i = buffer.get(); i > 0; i--) { skipString(buffer); @@ -28,24 +28,30 @@ inline void skipArray(IoBuffer &buffer) { } } -inline void skipString(IoBuffer &buffer) { skipArray(buffer); } +template +void skipArray(IoBuffer &buffer) { + buffer.skip(2 * static_cast(sizeof(int32_t))); + skipRawArray(buffer); +} + +inline void skipString(IoBuffer &buffer) { skipRawArray(buffer); } template -inline void skipMultiArray(IoBuffer &buffer) { +void skipMultiArray(IoBuffer &buffer) { buffer.skip(buffer.get() * static_cast(sizeof(int32_t))); // skip size header - skipArray(buffer); // skip elements + skipRawArray(buffer); // skip elements } template -inline int getTypeId() { +int getTypeId() { return IoSerialiser::getDataTypeId(); } template -inline int getTypeIdVector() { +int getTypeIdVector() { return IoSerialiser>::getDataTypeId(); } template -inline int getTypeIdMultiArray() { +int getTypeIdMultiArray() { return IoSerialiser>::getDataTypeId(); } @@ -53,12 +59,12 @@ inline int getTypeIdMultiArray() { template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFF; } // default value + static constexpr uint8_t getDataTypeId() { return 0xFF; } // default value }; template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v) { return 0; } else if constexpr (std::is_same_v) { return 1; } @@ -70,6 +76,7 @@ struct IoSerialiser { else if constexpr (std::is_same_v) { return 201; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const T &value) noexcept { @@ -83,7 +90,7 @@ struct IoSerialiser { template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 7; } + static constexpr uint8_t getDataTypeId() { return 7; } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) noexcept { buffer.put(value); @@ -96,9 +103,9 @@ struct IoSerialiser { template struct IoSerialiser { - using MemberType = typename T::value_type; + using MemberType = T::value_type; - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v) { return 9; } else if constexpr (std::is_same_v) { return 10; } @@ -111,20 +118,25 @@ struct IoSerialiser { else if constexpr (opencmw::is_stringlike ) { return 16; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const T &value) noexcept { + buffer.put(1); + buffer.put(static_cast(value.size())); buffer.put(value); } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, T &value) noexcept { - buffer.getArray(value); + const int nx = buffer.get(); + const int ny = buffer.get(); + buffer.getArray(value, nx * ny); } }; template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v && T::n_dims_ == 1) { return cmwlight::getTypeIdVector(); } else if constexpr (std::is_same_v && T::n_dims_ == 1) { return cmwlight::getTypeIdVector(); } @@ -155,6 +167,7 @@ struct IoSerialiser { else if constexpr (opencmw::is_stringlike ) { return 32; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const T &value) noexcept { @@ -169,7 +182,7 @@ struct IoSerialiser { constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, T &value) noexcept { const std::array dimWire = buffer.getArray(); for (auto i = 0U; i < T::n_dims_; i++) { - value.dimensions()[i] = static_cast(dimWire[i]); + value.dimensions()[i] = static_cast(dimWire[i]); } value.element_count() = value.dimensions()[T::n_dims_ - 1]; value.stride(T::n_dims_ - 1) = 1; @@ -185,7 +198,7 @@ struct IoSerialiser { template struct IoSerialiser> { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { // clang-format off if constexpr (std::is_same_v) { return 9; } else if constexpr (std::is_same_v) { return 10; } @@ -198,6 +211,7 @@ struct IoSerialiser> { else if constexpr (opencmw::is_stringlike ) { return 16; } else { static_assert(opencmw::always_false); } // clang-format on + return 0xff; // never reached but suppresses control flow warnings } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, const std::set &value) noexcept { @@ -235,7 +249,7 @@ struct IoSerialiser> { template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0x08; } + static constexpr uint8_t getDataTypeId() { return 0x08; } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const START_MARKER &/*value*/) noexcept { buffer.put(static_cast(field.subfields)); @@ -249,7 +263,7 @@ struct IoSerialiser { template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFE; } + static constexpr uint8_t getDataTypeId() { return 0xFE; } static void serialise(IoBuffer &/*buffer*/, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -262,7 +276,7 @@ struct IoSerialiser { template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 0xFD; } + static constexpr uint8_t getDataTypeId() { return 0xFD; } static void serialise(IoBuffer &/*buffer*/, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -283,6 +297,7 @@ struct IoSerialiser { else if (typeId == getTypeId()) { buffer.skip(sizeof(char )); } else if (typeId == getTypeId()) { skipString(buffer); } // arrays + else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } @@ -292,6 +307,7 @@ struct IoSerialiser { else if (typeId == getTypeIdVector()) { skipArray(buffer); } else if (typeId == getTypeIdVector()) { skipArray(buffer); } // 2D and Multi array + else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } else if (typeId == getTypeIdMultiArray() || typeId == getTypeIdMultiArray()) { skipMultiArray(buffer); } @@ -331,10 +347,10 @@ struct FieldHeaderWriter { template<> struct FieldHeaderReader { template - inline static void get(IoBuffer &buffer, DeserialiserInfo & /*info*/, FieldDescription auto &field) { + static void get(IoBuffer &buffer, DeserialiserInfo & /*info*/, FieldDescription auto &field) { field.headerStart = buffer.position(); field.dataEndPosition = std::numeric_limits::max(); - field.modifier = ExternalModifier::UNKNOWN; + field.modifier = UNKNOWN; if (field.subfields == 0) { if (field.hierarchyDepth != 0 && field.intDataType == IoSerialiser::getDataTypeId() && buffer.get() != 0) { throw ProtocolException("logic error, parent serialiser claims no data but header differs"); @@ -378,7 +394,7 @@ inline DeserialiserInfo checkHeaderInfo(IoBuffer &buffer, Deserialiser */ template struct IoSerialiser> { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); // this is just a stand-in and will be overwritten with the actual type id in the serialise function } @@ -401,7 +417,7 @@ struct IoSerialiser> { template struct IoSerialiser> { - inline static constexpr uint8_t getDataTypeId() { + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } @@ -409,14 +425,14 @@ struct IoSerialiser> { buffer.put(static_cast(value.size())); for (auto &[key, val] : value) { if constexpr (isReflectableClass()) { // nested data-structure - const auto subfields = value.size(); - FieldDescription auto field = newFieldHeader(buffer, parentField.hierarchyDepth + 1, FWD(val), subfields); - const std::size_t posSizePositionStart = FieldHeaderWriter::template put(buffer, field, val); - const std::size_t posStartDataStart = buffer.size(); + const auto subfields = value.size(); + FieldDescription auto field = newFieldHeader(buffer, parentField.hierarchyDepth + 1, FWD(val), subfields); + [[maybe_unused]] const std::size_t posSizePositionStart = FieldHeaderWriter::put(buffer, field, val); + [[maybe_unused]] const std::size_t posStartDataStart = buffer.size(); return; } else { // field is a (possibly annotated) primitive type - FieldDescription auto field = opencmw::detail::newFieldHeader(buffer, key.c_str(), parentField.hierarchyDepth + 1, val, 0); - FieldHeaderWriter::template put(buffer, field, val); + FieldDescription auto field = detail::newFieldHeader(buffer, key.c_str(), parentField.hierarchyDepth + 1, val, 0); + FieldHeaderWriter::put(buffer, field, val); } } } @@ -425,17 +441,17 @@ struct IoSerialiser> { DeserialiserInfo info; constexpr ProtocolCheck check = ProtocolCheck::IGNORE; using protocol = CmwLight; - auto field = opencmw::detail::newFieldHeader(buffer, "", parent.hierarchyDepth, ValueType{}, parent.subfields); + auto field = detail::newFieldHeader(buffer, "", parent.hierarchyDepth, ValueType{}, parent.subfields); while (buffer.position() < buffer.size()) { auto previousSubFields = field.subfields; - FieldHeaderReader::template get(buffer, info, field); + FieldHeaderReader::get(buffer, info, field); buffer.set_position(field.dataStartPosition); // skip to data start if (field.intDataType == IoSerialiser::getDataTypeId()) { // reached end of sub-structure try { IoSerialiser::deserialise(buffer, field, END_MARKER_INST); } catch (...) { - if (opencmw::detail::handleDeserialisationErrorAndSkipToNextField(buffer, field, info, "IoSerialiser<{}, END_MARKER>::deserialise(buffer, {}::{}, END_MARKER_INST): position {} vs. size {} -- exception: {}", + if (detail::handleDeserialisationErrorAndSkipToNextField(buffer, field, info, "IoSerialiser<{}, END_MARKER>::deserialise(buffer, {}::{}, END_MARKER_INST): position {} vs. size {} -- exception: {}", protocol::protocolName(), parent.fieldName, field.fieldName, buffer.position(), buffer.size(), what())) { continue; } @@ -454,8 +470,8 @@ struct IoSerialiser> { } else { constexpr int requestedType = IoSerialiser::getDataTypeId(); if (requestedType != field.intDataType) { // mismatching data-type - opencmw::detail::moveToFieldEndBufferPosition(buffer, field); - opencmw::detail::handleDeserialisationError(info, "mismatched field type for map field {} - requested type: {} (typeID: {}) got: {}", field.fieldName, typeName, requestedType, field.intDataType); + detail::moveToFieldEndBufferPosition(buffer, field); + detail::handleDeserialisationError(info, "mismatched field type for map field {} - requested type: {} (typeID: {}) got: {}", field.fieldName, typeName, requestedType, field.intDataType); return; } IoSerialiser::deserialise(buffer, field, fieldValue->second); diff --git a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp index 6e417a38..e83c5f25 100644 --- a/src/serialiser/test/IoSerialiserCmwLight_tests.cpp +++ b/src/serialiser/test/IoSerialiserCmwLight_tests.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -35,7 +34,7 @@ struct SimpleTestData { std::unique_ptr e = nullptr; std::set f{ "one", "two", "three" }; std::map g{ { "g1", 1 }, { "g2", 2 }, { "g3", 3 } }; - bool operator==(const ioserialiser_cmwlight_test::SimpleTestData &o) const { // deep comparison function + bool operator==(const SimpleTestData &o) const { // deep comparison function return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; } }; @@ -61,10 +60,10 @@ struct SimpleTestDataMapAsNested { std::unique_ptr e = nullptr; std::set f{ "one", "two", "three" }; SimpleTestDataMapNested g{ 1, 2, 3 }; - bool operator==(const ioserialiser_cmwlight_test::SimpleTestDataMapAsNested &o) const { // deep comparison function + bool operator==(const SimpleTestDataMapAsNested &o) const { // deep comparison function return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; } - bool operator==(const ioserialiser_cmwlight_test::SimpleTestData &o) const { // deep comparison function + bool operator==(const SimpleTestData &o) const { // deep comparison function return a == o.a && ab == o.ab && abc == o.abc && b == o.b && c == o.c && cd == o.cd && d == o.d && ((!e && !o.e) || *e == *(o.e)) && f == o.f && g == o.g; } }; @@ -74,7 +73,7 @@ ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMapAsNested, a, ENABLE_REFLECTION_FOR(ioserialiser_cmwlight_test::SimpleTestDataMapNested, g1, g2, g3) // small utility function that prints the content of a string in the classic hexedit way with address, hexadecimal and ascii representations -static std::string hexview(const std::string_view value, std::size_t bytesPerLine = 4) { +static std::string hexview(const std::string_view value, const std::size_t bytesPerLine = 4) { std::string result; result.reserve(value.size() * 4); std::string alpha; // temporarily store the ascii representation @@ -172,8 +171,8 @@ TEST_CASE("IoClassSerialiserCmwLight simple test", "[IoClassSerialiser]") { std::cout << std::format("object (std::format): {}\n", data); std::cout << "object (long): " << ClassInfoVerbose << data << '\n'; - opencmw::serialise(buffer, data); - opencmw::serialise(bufferMap, dataMap); + opencmw::serialise(buffer, data); + opencmw::serialise(bufferMap, dataMap); std::cout << std::format("buffer size (after): {} bytes\n", buffer.size()); buffer.put("a\0df"sv); // add some garbage after the serialised object to check if it is handled correctly @@ -256,9 +255,9 @@ TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { }; SimpleTestDataMoreFields data2; std::cout << std::format("object (std::format): {}\n", data); - opencmw::serialise(buffer, data); + opencmw::serialise(buffer, data); buffer.reset(); - auto result = opencmw::deserialise(buffer, data2); + auto result = opencmw::deserialise(buffer, data2); std::cout << std::format("deserialised object (std::format): {}\n", data2); std::cout << "deserialisation messages: " << result << std::endl; REQUIRE(result.setFields["root"] == std::vector{ 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 }); @@ -267,9 +266,9 @@ TEST_CASE("IoClassSerialiserCmwLight missing field", "[IoClassSerialiser]") { std::cout << "\nand now the other way round!\n\n"; buffer.clear(); - opencmw::serialise(buffer, data2); + opencmw::serialise(buffer, data2); buffer.reset(); - auto result_back = opencmw::deserialise(buffer, data); + auto result_back = opencmw::deserialise(buffer, data); std::cout << std::format("deserialised object (std::format): {}\n", data); std::cout << "deserialisation messages: " << result_back << std::endl; REQUIRE(result_back.setFields["root"] == std::vector{ 1, 1, 1, 1, 1, 1, 1, 1, 0, 1, 1 }); @@ -297,7 +296,7 @@ TEST_CASE("IoClassSerialiserCmwLight deserialise into map", "[IoClassSerialiser] // serialise IoBuffer buffer; IntegerMap input{ 23, 13, 37 }; - opencmw::serialise(buffer, input); + opencmw::serialise(buffer, input); buffer.reset(); REQUIRE(buffer.size() == sizeof(int32_t) /* map size */ + refl::reflect(input).members.size /* map entries */ * (sizeof(int32_t) /* string lengths */ + sizeof(uint8_t) /* type */ + sizeof(int32_t) /* int */) + 2 + 4 + 4 /* strings + \0 */); // std::cout << hexview(buffer.asString()); @@ -306,9 +305,9 @@ TEST_CASE("IoClassSerialiserCmwLight deserialise into map", "[IoClassSerialiser] std::map deserialised{}; DeserialiserInfo info; auto field = opencmw::detail::newFieldHeader(buffer, "map", 0, deserialised, -1); - opencmw::FieldHeaderReader::template get(buffer, info, field); + FieldHeaderReader::get(buffer, info, field); IoSerialiser::deserialise(buffer, field, START_MARKER_INST); - opencmw::IoSerialiser>::deserialise(buffer, field, deserialised); + IoSerialiser>::deserialise(buffer, field, deserialised); // check for correctness REQUIRE(deserialised.size() == 3); @@ -345,7 +344,7 @@ TEST_CASE("IoClassSerialiserCmwLight deserialise variant map", "[IoClassSerialis IoBuffer buffer; auto field = opencmw::detail::newFieldHeader(buffer, "map", 0, map, -1); - opencmw::IoSerialiser>>::serialise(buffer, field, map); + IoSerialiser>>::serialise(buffer, field, map); buffer.reset(); // std::print("expected:\n{}\ngot:\n{}\n", hexview(expected), hexview(buffer.asString())); @@ -389,10 +388,29 @@ struct DigitizerVersion { std::string gr_digitizer_version; std::string daqAPIVersion; }; +struct Empty {}; +struct StatusProperty { + int control; + std::vector detailedStatus; + std::vector detailedStatus_labels; + std::vector detailedStatus_severity; + std::vector error_codes; + std::vector error_cycle_names; + std::vector error_messages; + std::vector error_timestamps; + bool interlock; + bool modulesReady; + bool opReady; + int powerState; + int status; +}; } // namespace opencmw::serialiser::cmwlighttests ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::CmwLightHeaderOptions, b, e) ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::CmwLightHeader, x_2, x_0, x_1, f, x_7, d, x_3) ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::DigitizerVersion, classVersion, deployUnitVersion, fesaVersion, gr_flowgraph_version, gr_digitizer_version, daqAPIVersion) +ENABLE_REFLECTION_FOR(opencmw::serialiser::cmwlighttests::StatusProperty, control, detailedStatus, detailedStatus_labels, detailedStatus_severity, error_codes, error_cycle_names, error_messages, error_timestamps, interlock, modulesReady, opReady, powerState, status) +REFL_TYPE(opencmw::serialiser::cmwlighttests::Empty) +REFL_END TEST_CASE("IoClassSerialiserCmwLight Deserialise rda3 data", "[IoClassSerialiser]") { // ensure that important rda3 messages can be properly deserialized @@ -400,18 +418,20 @@ TEST_CASE("IoClassSerialiserCmwLight Deserialise rda3 data", "[IoClassSerialiser debug::resetStats(); using namespace opencmw::serialiser::cmwlighttests; { - std::string_view rda3ConnectReply = "\x07\x00\x00\x00\x02\x00\x00\x00\x30\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x31\x00\x07\x01\x00\x00\x00\x00\x02\x00\x00\x00\x32\x00\x01\x03\x02\x00\x00\x00\x33\x00\x08\x02\x00\x00\x00\x02\x00\x00\x00\x62\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x65\x00\x08\x00\x00\x00\x00\x02\x00\x00\x00\x37\x00\x01\x70\x02\x00\x00\x00\x64\x00\x07\x01\x00\x00\x00\x00\x02\x00\x00\x00\x66\x00\x07\x01\x00\x00\x00\x00"sv; - // 0 1 2 3 4 5 6 7 8 9 a b c d e f - // 000 "\x07\x00\x00\x00\x02\x00\x00\x00" "\x30\x00\x04\x00\x00\x00\x00\x00" - // 010 "\x00\x00\x00\x02\x00\x00\x00\x31" "\x00\x07\x01\x00\x00\x00\x00\x02" - // 020 "\x00\x00\x00\x32\x00\x01\x03\x02" "\x00\x00\x00\x33\x00\x08\x02\x00" - // 030 "\x00\x00\x02\x00\x00\x00\x62\x00" "\x04\x00\x00\x00\x00\x00\x00\x00" - // 040 "\x00\x02\x00\x00\x00\x65\x00\x08" "\x00\x00\x00\x00\x02\x00\x00\x00" - // 050 "\x37\x00\x01\x70\x02\x00\x00\x00" "\x64\x00\x07\x01\x00\x00\x00\x00" - // 060 "\x02\x00\x00\x00\x66\x00\x07\x01" "\x00\x00\x00\x00"sv + std::vector rda3ConnectReply = { // + 0x07, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, /**/ 0x30, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x31, /**/ 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x00, 0x02, // + 0x00, 0x00, 0x00, 0x32, 0x00, 0x01, 0x03, 0x02, /**/ 0x00, 0x00, 0x00, 0x33, 0x00, 0x08, 0x02, 0x00, // + 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x62, 0x00, /**/ 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // + 0x00, 0x02, 0x00, 0x00, 0x00, 0x65, 0x00, 0x08, /**/ 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, // + 0x37, 0x00, 0x01, 0x70, 0x02, 0x00, 0x00, 0x00, /**/ 0x64, 0x00, 0x07, 0x01, 0x00, 0x00, 0x00, 0x00, // + 0x02, 0x00, 0x00, 0x00, 0x66, 0x00, 0x07, 0x01, /**/ 0x00, 0x00, 0x00, 0x00 }; // IoBuffer buffer{ rda3ConnectReply.data(), rda3ConnectReply.size() }; CmwLightHeader deserialised; - auto result = opencmw::deserialise(buffer, deserialised); + auto result = opencmw::deserialise(buffer, deserialised); + REQUIRE(result.additionalFields.empty()); + REQUIRE(result.exceptions.empty()); + REQUIRE(result.setFields["root"sv].size() == 7); REQUIRE(deserialised.requestType() == 3); REQUIRE(deserialised.id() == 0); @@ -423,68 +443,123 @@ TEST_CASE("IoClassSerialiserCmwLight Deserialise rda3 data", "[IoClassSerialiser REQUIRE(deserialised.options()->sessionBody.empty()); } { - std::string_view data = "\x07\x00\x00\x00\x02\x00\x00\x00\x30\x00\x04\x09\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x31\x00\x07\x08\x00\x00\x00\x47\x53\x43\x44\x30\x30\x32\x00\x02\x00\x00\x00\x32\x00\x01\x0b\x02\x00\x00\x00\x33\x00\x08\x01\x00\x00\x00\x02\x00\x00\x00\x65\x00\x08\x00\x00\x00\x00\x02\x00\x00\x00\x37\x00\x01\x00\x02\x00\x00\x00\x64\x00\x07\x25\x01\x00\x00\x52\x65\x6d\x6f\x74\x65\x48\x6f\x73\x74\x49\x6e\x66\x6f\x49\x6d\x70\x6c\x5b\x6e\x61\x6d\x65\x3d\x66\x65\x73\x61\x2d\x65\x78\x70\x6c\x6f\x72\x65\x72\x2d\x61\x70\x70\x3b\x20\x75\x73\x65\x72\x4e\x61\x6d\x65\x3d\x61\x6b\x72\x69\x6d\x6d\x3b\x20\x61\x70\x70\x49\x64\x3d\x5b\x61\x70\x70\x3d\x66\x65\x73\x61\x2d\x65\x78\x70\x6c\x6f\x72\x65\x72\x2d\x61\x70\x70\x3b\x76\x65\x72\x3d\x31\x39\x2e\x30\x2e\x30\x3b\x75\x69\x64\x3d\x61\x6b\x72\x69\x6d\x6d\x3b\x68\x6f\x73\x74\x3d\x53\x59\x53\x50\x43\x30\x30\x38\x3b\x70\x69\x64\x3d\x31\x39\x31\x36\x31\x36\x3b\x5d\x3b\x20\x70\x72\x6f\x63\x65\x73\x73\x3d\x66\x65\x73\x61\x2d\x65\x78\x70\x6c\x6f\x72\x65\x72\x2d\x61\x70\x70\x3b\x20\x70\x69\x64\x3d\x31\x39\x31\x36\x31\x36\x3b\x20\x61\x64\x64\x72\x65\x73\x73\x3d\x74\x63\x70\x3a\x2f\x2f\x53\x59\x53\x50\x43\x30\x30\x38\x3a\x30\x3b\x20\x73\x74\x61\x72\x74\x54\x69\x6d\x65\x3d\x32\x30\x32\x34\x2d\x30\x37\x2d\x30\x34\x20\x31\x31\x3a\x31\x31\x3a\x31\x32\x3b\x20\x63\x6f\x6e\x6e\x65\x63\x74\x69\x6f\x6e\x54\x69\x6d\x65\x3d\x41\x62\x6f\x75\x74\x20\x61\x67\x6f\x3b\x20\x76\x65\x72\x73\x69\x6f\x6e\x3d\x31\x30\x2e\x33\x2e\x30\x3b\x20\x6c\x61\x6e\x67\x75\x61\x67\x65\x3d\x4a\x61\x76\x61\x5d\x31\x00\x02\x00\x00\x00\x66\x00\x07\x08\x00\x00\x00\x56\x65\x72\x73\x69\x6f\x6e\x00"sv; // reply req type: session confirm - // \x07 \x00 \x00 \x00 .... - // \x02 \x00 \x00 \x00 \x30 \x00 \x04 \x09 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x02 ....0... ........ - // \x00 \x00 \x00 \x31 \x00 \x07 \x08 \x00 \x00 \x00 \x47 \x53 \x43 \x44 \x30 \x30 ...1.... ..GSCD00 - // \x32 \x00 \x02 \x00 \x00 \x00 \x32 \x00 \x01 \x0b \x02 \x00 \x00 \x00 \x33 \x00 2.....2. ......3. - // \x08 \x01 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x65 \x00 \x08 \x00 \x00 \x00 \x00 ........ .e...... - // \x02 \x00 \x00 \x00 \x37 \x00 \x01 \x00 \x02 \x00 \x00 \x00 \x64 \x00 \x07 \x25 ....7... ....d..% - // \x01 \x00 \x00 \x52 \x65 \x6d \x6f \x74 \x65 \x48 \x6f \x73 \x74 \x49 \x6e \x66 ...Remot eHostInf - // \x6f \x49 \x6d \x70 \x6c \x5b \x6e \x61 \x6d \x65 \x3d \x66 \x65 \x73 \x61 \x2d oImpl[na me=fesa- - // \x65 \x78 \x70 \x6c \x6f \x72 \x65 \x72 \x2d \x61 \x70 \x70 \x3b \x20 \x75 \x73 explorer -app; us - // \x65 \x72 \x4e \x61 \x6d \x65 \x3d \x61 \x6b \x72 \x69 \x6d \x6d \x3b \x20 \x61 erName=a krimm; a - // \x70 \x70 \x49 \x64 \x3d \x5b \x61 \x70 \x70 \x3d \x66 \x65 \x73 \x61 \x2d \x65 ppId=[ap p=fesa-e - // \x78 \x70 \x6c \x6f \x72 \x65 \x72 \x2d \x61 \x70 \x70 \x3b \x76 \x65 \x72 \x3d xplorer- app;ver= - // \x31 \x39 \x2e \x30 \x2e \x30 \x3b \x75 \x69 \x64 \x3d \x61 \x6b \x72 \x69 \x6d 19.0.0;u id=akrim - // \x6d \x3b \x68 \x6f \x73 \x74 \x3d \x53 \x59 \x53 \x50 \x43 \x30 \x30 \x38 \x3b m;host=S YSPC008; - // \x70 \x69 \x64 \x3d \x31 \x39 \x31 \x36 \x31 \x36 \x3b \x5d \x3b \x20 \x70 \x72 pid=1916 16;]; pr - // \x6f \x63 \x65 \x73 \x73 \x3d \x66 \x65 \x73 \x61 \x2d \x65 \x78 \x70 \x6c \x6f ocess=fe sa-explo - // \x72 \x65 \x72 \x2d \x61 \x70 \x70 \x3b \x20 \x70 \x69 \x64 \x3d \x31 \x39 \x31 rer-app; pid=191 - // \x36 \x31 \x36 \x3b \x20 \x61 \x64 \x64 \x72 \x65 \x73 \x73 \x3d \x74 \x63 \x70 616; add ress=tcp - // \x3a \x2f \x2f \x53 \x59 \x53 \x50 \x43 \x30 \x30 \x38 \x3a \x30 \x3b \x20 \x73 ://SYSPC 008:0; s - // \x74 \x61 \x72 \x74 \x54 \x69 \x6d \x65 \x3d \x32 \x30 \x32 \x34 \x2d \x30 \x37 tartTime =2024-07 - // \x2d \x30 \x34 \x20 \x31 \x31 \x3a \x31 \x31 \x3a \x31 \x32 \x3b \x20 \x63 \x6f -04 11:1 1:12; co - // \x6e \x6e \x65 \x63 \x74 \x69 \x6f \x6e \x54 \x69 \x6d \x65 \x3d \x41 \x62 \x6f nnection Time=Abo - // \x75 \x74 \x20 \x61 \x67 \x6f \x3b \x20 \x76 \x65 \x72 \x73 \x69 \x6f \x6e \x3d ut ago; version= - // \x31 \x30 \x2e \x33 \x2e \x30 \x3b \x20 \x6c \x61 \x6e \x67 \x75 \x61 \x67 \x65 10.3.0; language - // \x3d \x4a \x61 \x76 \x61 \x5d \x31 \x00 \x02 \x00 \x00 \x00 \x66 \x00 \x07 \x08 =Java]1. ....f... - // \x00 \x00 \x00 \x56 \x65 \x72 \x73 \x69 \x6f \x6e \x00 ...Versi on. + std::vector data = { 0x07, 0x00, 0x00, 0x00, // .... + 0x02, 0x00, 0x00, 0x00, 0x30, 0x00, 0x04, 0x09, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, // ....0... ........ + 0x00, 0x00, 0x00, 0x31, 0x00, 0x07, 0x08, 0x00, /**/ 0x00, 0x00, 0x47, 0x53, 0x43, 0x44, 0x30, 0x30, // ...1.... ..GSCD00 + 0x32, 0x00, 0x02, 0x00, 0x00, 0x00, 0x32, 0x00, /**/ 0x01, 0x0b, 0x02, 0x00, 0x00, 0x00, 0x33, 0x00, // 2.....2. ......3. + 0x08, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x65, 0x00, 0x08, 0x00, 0x00, 0x00, 0x00, // ........ .e...... + 0x02, 0x00, 0x00, 0x00, 0x37, 0x00, 0x01, 0x00, /**/ 0x02, 0x00, 0x00, 0x00, 0x64, 0x00, 0x07, 0x25, // ....7... ....d..% + 0x01, 0x00, 0x00, 0x52, 0x65, 0x6d, 0x6f, 0x74, /**/ 0x65, 0x48, 0x6f, 0x73, 0x74, 0x49, 0x6e, 0x66, // ...Remot eHostInf + 0x6f, 0x49, 0x6d, 0x70, 0x6c, 0x5b, 0x6e, 0x61, /**/ 0x6d, 0x65, 0x3d, 0x66, 0x65, 0x73, 0x61, 0x2d, // oImpl[na me=fesa- + 0x65, 0x78, 0x70, 0x6c, 0x6f, 0x72, 0x65, 0x72, /**/ 0x2d, 0x61, 0x70, 0x70, 0x3b, 0x20, 0x75, 0x73, // explorer -app; us + 0x65, 0x72, 0x4e, 0x61, 0x6d, 0x65, 0x3d, 0x61, /**/ 0x6b, 0x72, 0x69, 0x6d, 0x6d, 0x3b, 0x20, 0x61, // erName=a krimm; a + 0x70, 0x70, 0x49, 0x64, 0x3d, 0x5b, 0x61, 0x70, /**/ 0x70, 0x3d, 0x66, 0x65, 0x73, 0x61, 0x2d, 0x65, // ppId=[ap p=fesa-e + 0x78, 0x70, 0x6c, 0x6f, 0x72, 0x65, 0x72, 0x2d, /**/ 0x61, 0x70, 0x70, 0x3b, 0x76, 0x65, 0x72, 0x3d, // xplorer- app;ver= + 0x31, 0x39, 0x2e, 0x30, 0x2e, 0x30, 0x3b, 0x75, /**/ 0x69, 0x64, 0x3d, 0x61, 0x6b, 0x72, 0x69, 0x6d, // 19.0.0;u id=akrim + 0x6d, 0x3b, 0x68, 0x6f, 0x73, 0x74, 0x3d, 0x53, /**/ 0x59, 0x53, 0x50, 0x43, 0x30, 0x30, 0x38, 0x3b, // m;host=S YSPC008; + 0x70, 0x69, 0x64, 0x3d, 0x31, 0x39, 0x31, 0x36, /**/ 0x31, 0x36, 0x3b, 0x5d, 0x3b, 0x20, 0x70, 0x72, // pid=1916 16;]; pr + 0x6f, 0x63, 0x65, 0x73, 0x73, 0x3d, 0x66, 0x65, /**/ 0x73, 0x61, 0x2d, 0x65, 0x78, 0x70, 0x6c, 0x6f, // ocess=fe sa-explo + 0x72, 0x65, 0x72, 0x2d, 0x61, 0x70, 0x70, 0x3b, /**/ 0x20, 0x70, 0x69, 0x64, 0x3d, 0x31, 0x39, 0x31, // rer-app; pid=191 + 0x36, 0x31, 0x36, 0x3b, 0x20, 0x61, 0x64, 0x64, /**/ 0x72, 0x65, 0x73, 0x73, 0x3d, 0x74, 0x63, 0x70, // 616; add ress=tcp + 0x3a, 0x2f, 0x2f, 0x53, 0x59, 0x53, 0x50, 0x43, /**/ 0x30, 0x30, 0x38, 0x3a, 0x30, 0x3b, 0x20, 0x73, // ://SYSPC 008:0; s + 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65, /**/ 0x3d, 0x32, 0x30, 0x32, 0x34, 0x2d, 0x30, 0x37, // tartTime =2024-07 + 0x2d, 0x30, 0x34, 0x20, 0x31, 0x31, 0x3a, 0x31, /**/ 0x31, 0x3a, 0x31, 0x32, 0x3b, 0x20, 0x63, 0x6f, // -04 11:1 1:12; co + 0x6e, 0x6e, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, /**/ 0x54, 0x69, 0x6d, 0x65, 0x3d, 0x41, 0x62, 0x6f, // nnection Time=Abo + 0x75, 0x74, 0x20, 0x61, 0x67, 0x6f, 0x3b, 0x20, /**/ 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x3d, // ut ago; version= + 0x31, 0x30, 0x2e, 0x33, 0x2e, 0x30, 0x3b, 0x20, /**/ 0x6c, 0x61, 0x6e, 0x67, 0x75, 0x61, 0x67, 0x65, // 10.3.0; language + 0x3d, 0x4a, 0x61, 0x76, 0x61, 0x5d, 0x31, 0x00, /**/ 0x02, 0x00, 0x00, 0x00, 0x66, 0x00, 0x07, 0x08, // =Java]1. ....f... + 0x00, 0x00, 0x00, 0x56, 0x65, 0x72, 0x73, 0x69, /**/ 0x6f, 0x6e, 0x00 }; //...Versi on. IoBuffer buffer{ data.data(), data.size() }; CmwLightHeader deserialised; - auto result = opencmw::deserialise(buffer, deserialised); + auto result = opencmw::deserialise(buffer, deserialised); + REQUIRE(result.additionalFields.empty()); + REQUIRE(result.exceptions.empty()); + REQUIRE(result.setFields["root"sv].size() == 7); } { - std::string_view data = "\x06\x00\x00\x00\x02\x00\x00\x00\x30\x00\x04\x09\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x31\x00\x07\x01\x00\x00\x00\x00\x02\x00\x00\x00\x32\x00\x01\x03\x02\x00\x00\x00\x37\x00\x01\x00\x02\x00\x00\x00\x64\x00\x07\x01\x00\x00\x00\x00\x02\x00\x00\x00\x66\x00\x07\x01\x00\x00\x00\x00\x01\xc3\x06\x00\x00\x00\x0d\x00\x00\x00\x63\x6c\x61\x73\x73\x56\x65\x72\x73\x69\x6f\x6e\x00\x07\x06\x00\x00\x00\x36\x2e\x30\x2e\x30\x00\x0e\x00\x00\x00\x64\x61\x71\x41\x50\x49\x56\x65\x72\x73\x69\x6f\x6e\x00\x07\x04\x00\x00\x00\x32\x2e\x30\x00\x12\x00\x00\x00\x64\x65\x70\x6c\x6f\x79\x55\x6e\x69\x74\x56\x65\x72\x73\x69\x6f\x6e\x00\x07\x06\x00\x00\x00\x36\x2e\x30\x2e\x30\x00\x0c\x00\x00\x00\x66\x65\x73\x61\x56\x65\x72\x73\x69\x6f\x6e\x00\x07\x06\x00\x00\x00\x37\x2e\x33\x2e\x30\x00\x15\x00\x00\x00\x67\x72\x5f\x64\x69\x67\x69\x74\x69\x7a\x65\x72\x5f\x76\x65\x72\x73\x69\x6f\x6e\x00\x07\x08\x00\x00\x00\x35\x2e\x31\x2e\x34\x2e\x30\x00\x15\x00\x00\x00\x67\x72\x5f\x66\x6c\x6f\x77\x67\x72\x61\x70\x68\x5f\x76\x65\x72\x73\x69\x6f\x6e\x00\x07\x08\x00\x00\x00\x35\x2e\x30\x2e\x32\x2e\x30\x00\x01\x62\x03\x00\x00\x00\x02\x00\x00\x00\x35\x00\x04\x88\x39\xfe\x41\x88\xf7\xde\x17\x02\x00\x00\x00\x36\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\x00\x00\x78\x00\x08\x03\x00\x00\x00\x09\x00\x00\x00\x61\x63\x71\x53\x74\x61\x6d\x70\x00\x04\x88\x39\xfe\x41\x88\xf7\xde\x17\x05\x00\x00\x00\x74\x79\x70\x65\x00\x03\x02\x00\x00\x00\x08\x00\x00\x00\x76\x65\x72\x73\x69\x6f\x6e\x00\x03\x01\x00\x00\x00\x00\x03"; // Reply with Req Type = Reply, gets sent after get request - // \x06 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x30 \x00 \x04 ... .....0.. - // \x09 \x00 \x00 \x00 \x00 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x31 \x00 \x07 \x01 ........ ....1... - // \x00 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x32 \x00 \x01 \x03 \x02 \x00 \x00 \x00 ........ 2....... - // \x37 \x00 \x01 \x00 \x02 \x00 \x00 \x00 \x64 \x00 \x07 \x01 \x00 \x00 \x00 \x00 7....... d....... - // \x02 \x00 \x00 \x00 \x66 \x00 \x07 \x01 \x00 \x00 \x00 \x00 \x01 \xc3 \x06 \x00 ....f... ........ - // \x00 \x00 \x0d \x00 \x00 \x00 \x63 \x6c \x61 \x73 \x73 \x56 \x65 \x72 \x73 \x69 ......cl assVersi - // \x6f \x6e \x00 \x07 \x06 \x00 \x00 \x00 \x36 \x2e \x30 \x2e \x30 \x00 \x0e \x00 on...... 6.0.0... - // \x00 \x00 \x64 \x61 \x71 \x41 \x50 \x49 \x56 \x65 \x72 \x73 \x69 \x6f \x6e \x00 ..daqAPI Version. - // \x07 \x04 \x00 \x00 \x00 \x32 \x2e \x30 \x00 \x12 \x00 \x00 \x00 \x64 \x65 \x70 .....2.0 .....dep - // \x6c \x6f \x79 \x55 \x6e \x69 \x74 \x56 \x65 \x72 \x73 \x69 \x6f \x6e \x00 \x07 loyUnitV ersion.. - // \x06 \x00 \x00 \x00 \x36 \x2e \x30 \x2e \x30 \x00 \x0c \x00 \x00 \x00 \x66 \x65 ....6.0. 0.....fe - // \x73 \x61 \x56 \x65 \x72 \x73 \x69 \x6f \x6e \x00 \x07 \x06 \x00 \x00 \x00 \x37 saVersio n......7 - // \x2e \x33 \x2e \x30 \x00 \x15 \x00 \x00 \x00 \x67 \x72 \x5f \x64 \x69 \x67 \x69 .3.0.... .gr_digi - // \x74 \x69 \x7a \x65 \x72 \x5f \x76 \x65 \x72 \x73 \x69 \x6f \x6e \x00 \x07 \x08 tizer_ve rsion... - // \x00 \x00 \x00 \x35 \x2e \x31 \x2e \x34 \x2e \x30 \x00 \x15 \x00 \x00 \x00 \x67 ...5.1.4 .0.....g - // \x72 \x5f \x66 \x6c \x6f \x77 \x67 \x72 \x61 \x70 \x68 \x5f \x76 \x65 \x72 \x73 r_flowgr aph_vers - // \x69 \x6f \x6e \x00 \x07 \x08 \x00 \x00 \x00 \x35 \x2e \x30 \x2e \x32 \x2e \x30 ion..... .5.0.2.0 - // \x00 \x01 \x62 \x03 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x35 \x00 \x04 \x88 \x39 ..b..... ...5...9 - // \xfe \x41 \x88 \xf7 \xde \x17 \x02 \x00 \x00 \x00 \x36 \x00 \x04 \x00 \x00 \x00 .A...... ..6..... - // \x00 \x00 \x00 \x00 \x00 \x02 \x00 \x00 \x00 \x78 \x00 \x08 \x03 \x00 \x00 \x00 ........ .x...... - // \x09 \x00 \x00 \x00 \x61 \x63 \x71 \x53 \x74 \x61 \x6d \x70 \x00 \x04 \x88 \x39 ....acqS tamp...9 - // \xfe \x41 \x88 \xf7 \xde \x17 \x05 \x00 \x00 \x00 \x74 \x79 \x70 \x65 \x00 \x03 .A...... ..type.. - // \x02 \x00 \x00 \x00 \x08 \x00 \x00 \x00 \x76 \x65 \x72 \x73 \x69 \x6f \x6e \x00 ........ version. - // \x03 \x01 \x00 \x00 \x00 \x00 \x03 ....... + std::vector data = { 0x06, 0x00, // .. + 0x00, 0x00, 0x0d, 0x00, 0x00, 0x00, 0x63, 0x6c, /**/ 0x61, 0x73, 0x73, 0x56, 0x65, 0x72, 0x73, 0x69, // ......cl assVersi + 0x6f, 0x6e, 0x00, 0x07, 0x06, 0x00, 0x00, 0x00, /**/ 0x36, 0x2e, 0x30, 0x2e, 0x30, 0x00, 0x0e, 0x00, // on...... 6.0.0... + 0x00, 0x00, 0x64, 0x61, 0x71, 0x41, 0x50, 0x49, /**/ 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, // ..daqAPI Version. + 0x07, 0x04, 0x00, 0x00, 0x00, 0x32, 0x2e, 0x30, /**/ 0x00, 0x12, 0x00, 0x00, 0x00, 0x64, 0x65, 0x70, // .....2.0 .....dep + 0x6c, 0x6f, 0x79, 0x55, 0x6e, 0x69, 0x74, 0x56, /**/ 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x07, // loyUnitV ersion.. + 0x06, 0x00, 0x00, 0x00, 0x36, 0x2e, 0x30, 0x2e, /**/ 0x30, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x66, 0x65, // ....6.0. 0.....fe + 0x73, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, /**/ 0x6e, 0x00, 0x07, 0x06, 0x00, 0x00, 0x00, 0x37, // saVersio n......7 + 0x2e, 0x33, 0x2e, 0x30, 0x00, 0x15, 0x00, 0x00, /**/ 0x00, 0x67, 0x72, 0x5f, 0x64, 0x69, 0x67, 0x69, // .3.0.... .gr_digi + 0x74, 0x69, 0x7a, 0x65, 0x72, 0x5f, 0x76, 0x65, /**/ 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, 0x07, 0x08, // tizer_ve rsion... + 0x00, 0x00, 0x00, 0x35, 0x2e, 0x31, 0x2e, 0x34, /**/ 0x2e, 0x30, 0x00, 0x15, 0x00, 0x00, 0x00, 0x67, // ...5.1.4 .0.....g + 0x72, 0x5f, 0x66, 0x6c, 0x6f, 0x77, 0x67, 0x72, /**/ 0x61, 0x70, 0x68, 0x5f, 0x76, 0x65, 0x72, 0x73, // r_flowgr aph_vers + 0x69, 0x6f, 0x6e, 0x00, 0x07, 0x08, 0x00, 0x00, /**/ 0x00, 0x35, 0x2e, 0x30, 0x2e, 0x32, 0x2e, 0x30, // ion..... .5.0.2.0 + 0x00, 0x01, 0x62, 0x03, 0x00, 0x00, 0x00, 0x02, /**/ 0x00, 0x00, 0x00, 0x35, 0x00, 0x04, 0x88, 0x39, // ..b..... ...5...9 + 0xfe, 0x41, 0x88, 0xf7, 0xde, 0x17, 0x02, 0x00, /**/ 0x00, 0x00, 0x36, 0x00, 0x04, 0x00, 0x00, 0x00, // .A...... ..6..... + 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x78, 0x00, 0x08, 0x03, 0x00, 0x00, 0x00, // ........ .x...... + 0x09, 0x00, 0x00, 0x00, 0x61, 0x63, 0x71, 0x53, /**/ 0x74, 0x61, 0x6d, 0x70, 0x00, 0x04, 0x88, 0x39, // ....acqS tamp...9 + 0xfe, 0x41, 0x88, 0xf7, 0xde, 0x17, 0x05, 0x00, /**/ 0x00, 0x00, 0x74, 0x79, 0x70, 0x65, 0x00, 0x03, // .A...... ..type.. + 0x02, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, /**/ 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x00, // ........ version. + 0x03, 0x01, 0x00, 0x00, 0x00, 0x00, 0x03} ; // ....... IoBuffer buffer{ data.data(), data.size() }; DigitizerVersion deserialised; - auto result = opencmw::deserialise(buffer, deserialised); + auto result = opencmw::deserialise(buffer, deserialised); + REQUIRE(result.additionalFields.empty()); + REQUIRE(result.exceptions.empty()); + REQUIRE(result.setFields["root"sv].size() == 6); + } + { + std::vector data{ // + 0x0d, 0x00, 0x00, 0x00, 0x08, 0x00, 0x00, 0x00, /**/ 0x63, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c, 0x00, /**/ 0x03, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, /**/ 0x00, 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, // ........ control. ........ .detaile + 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x00, /**/ 0x09, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x02, 0x00, 0x00, 0x00, 0x01, 0x01, 0x16, /**/ 0x00, 0x00, 0x00, 0x64, 0x65, 0x74, 0x61, 0x69, // dStatus. ........ ........ ...detai + 0x6c, 0x65, 0x64, 0x53, 0x74, 0x61, 0x74, 0x75, /**/ 0x73, 0x5f, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x73, /**/ 0x00, 0x10, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, /**/ 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x0f, 0x00, // ledStatu s_labels ........ ........ + 0x00, 0x00, 0x6d, 0x79, 0x53, 0x74, 0x61, 0x74, /**/ 0x75, 0x73, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x31, /**/ 0x00, 0x0f, 0x00, 0x00, 0x00, 0x6d, 0x79, 0x53, /**/ 0x74, 0x61, 0x74, 0x75, 0x73, 0x4c, 0x61, 0x62, // ..myStat usLabel1 .....myS tatusLab + 0x65, 0x6c, 0x32, 0x00, 0x18, 0x00, 0x00, 0x00, /**/ 0x64, 0x65, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x64, /**/ 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x5f, 0x73, /**/ 0x65, 0x76, 0x65, 0x72, 0x69, 0x74, 0x79, 0x00, // el2..... detailed Status_s everity. + 0x0c, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, /**/ 0x00, 0x02, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, /**/ 0x00, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, // ........ ........ ........ .error_c + 0x6f, 0x64, 0x65, 0x73, 0x00, 0x0c, 0x01, 0x00, /**/ 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x10, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // odes.... ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x12, 0x00, 0x00, 0x00, 0x65, 0x72, /**/ 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x79, 0x63, 0x6c, // ........ ........ ......er ror_cycl + 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x00, /**/ 0x10, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, /**/ 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /**/ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, // e_names. ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, /**/ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /**/ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, /**/ 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, /**/ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, /**/ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /**/ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x01, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x00, 0x00, /**/ 0x00, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, /**/ 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x73, 0x00, /**/ 0x10, 0x01, 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, // ........ .error_m essages. ........ + 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /**/ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, /**/ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, /**/ 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, /**/ 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, /**/ 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, /**/ 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, /**/ 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, /**/ 0x01, 0x00, 0x00, 0x00, 0x00, 0x11, 0x00, 0x00, /**/ 0x00, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x74, // ........ ........ ........ .error_t + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, /**/ 0x73, 0x00, 0x0d, 0x01, 0x00, 0x00, 0x00, 0x10, /**/ 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // imestamp s....... ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // ........ ........ ........ ........ + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, /**/ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, /**/ 0x00, 0x00, 0x00, 0x69, 0x6e, 0x74, 0x65, 0x72, // ........ ........ ........ ...inter + 0x6c, 0x6f, 0x63, 0x6b, 0x00, 0x00, 0x00, 0x0d, /**/ 0x00, 0x00, 0x00, 0x6d, 0x6f, 0x64, 0x75, 0x6c, /**/ 0x65, 0x73, 0x52, 0x65, 0x61, 0x64, 0x79, 0x00, /**/ 0x00, 0x01, 0x08, 0x00, 0x00, 0x00, 0x6f, 0x70, // lock.... ...modul esReady. ......op + 0x52, 0x65, 0x61, 0x64, 0x79, 0x00, 0x00, 0x01, /**/ 0x0b, 0x00, 0x00, 0x00, 0x70, 0x6f, 0x77, 0x65, /**/ 0x72, 0x53, 0x74, 0x61, 0x74, 0x65, 0x00, 0x03, /**/ 0x01, 0x00, 0x00, 0x00, 0x07, 0x00, 0x00, 0x00, // Ready... ....powe rState.. ........ + 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x00, 0x03, /**/ 0x01, 0x00, 0x00, 0x00}; // status.. .... + IoBuffer buffer{ data.data(), data.size() }; + REQUIRE(buffer.size() == 748); + // deserialise once into an empty struct to verify that the fields can be correctly skipped and end up in the deserialiserInfo + Empty empty; + auto result = opencmw::deserialise(buffer, empty); + REQUIRE(result.additionalFields.size() == 13); // [root::control, root::detailedStatus, root::detailedStatus_labels, root::detailedStatus_severity, root::error_codes, root::error_cycle_names, root::error_messages, root::error_timestamps, root::interlock, root::modulesReady, root::opReady, root::powerState, root::status] [3, 9, 16, 12, 12, 16, 16, 13, 0, 0, 0, 3, 3] + REQUIRE(result.exceptions.size() == 13); // each missing field also produces an excception + REQUIRE(result.setFields["root"].empty()); + // deserialise into the correct domain object + buffer.reset(); + StatusProperty status; + auto result2 = opencmw::deserialise(buffer, status); + REQUIRE(result2.additionalFields.empty()); + REQUIRE(result2.exceptions.empty()); + REQUIRE(result2.setFields["root"sv].size() == 13); + REQUIRE(status.control == 0); + REQUIRE(status.detailedStatus == std::vector{true, true}); + REQUIRE(status.detailedStatus_labels == std::vector{"myStatusLabel1"s, "myStatusLabel2"s}); + REQUIRE(status.detailedStatus_severity == std::vector{0, 0}); + REQUIRE(status.error_codes.size() == 16); + REQUIRE(status.error_cycle_names.size() == 16); + REQUIRE(status.error_messages.size() == 16); + REQUIRE(status.error_timestamps.size() == 16); + REQUIRE_FALSE(status.interlock); + REQUIRE(status.modulesReady); + REQUIRE(status.opReady); + REQUIRE(status.powerState == 1); + REQUIRE(status.status == 1); } REQUIRE(opencmw::debug::dealloc == opencmw::debug::alloc); // a memory leak occurred debug::resetStats(); From c7436490f66f03354bc205c13300aa07f0ab48d8 Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Wed, 18 Feb 2026 16:48:27 +0100 Subject: [PATCH 6/7] JsonSerialiser: allow serialising list of objects Allows (de)serialising lists of reflectable objects. Due to the current structure of the code, some error reporting features are not supported for nested objects. Also fixes an off by one error which consumes one additional byte from the buffer after the object. Signed-off-by: Alexander Krimm --- src/serialiser/include/IoSerialiserJson.hpp | 132 ++++++++++-------- .../test/IoSerialiserJson_tests.cpp | 3 +- 2 files changed, 75 insertions(+), 60 deletions(-) diff --git a/src/serialiser/include/IoSerialiserJson.hpp b/src/serialiser/include/IoSerialiserJson.hpp index b546c51c..a99220b4 100644 --- a/src/serialiser/include/IoSerialiserJson.hpp +++ b/src/serialiser/include/IoSerialiserJson.hpp @@ -15,10 +15,10 @@ struct Json : Protocol<"Json"> {}; // as specified in https://www.json.org/json- namespace json { -constexpr inline bool isNumberChar(uint8_t c) { return (c >= '+' && c <= '9' && c != '/' && c != ',') || c == 'e' || c == 'E'; } -constexpr inline bool isWhitespace(uint8_t c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } -constexpr inline bool isHexNumberChar(int8_t c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } -constexpr inline bool isHexNumber(const char *start, const size_t size) noexcept { +constexpr bool isNumberChar(uint8_t c) { return (c >= '+' && c <= '9' && c != '/' && c != ',') || c == 'e' || c == 'E'; } +constexpr bool isWhitespace(uint8_t c) { return c == ' ' || c == '\t' || c == '\n' || c == '\r'; } +constexpr bool isHexNumberChar(int8_t c) { return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F'); } +constexpr bool isHexNumber(const char *start, const size_t size) noexcept { for (size_t i = 0; i < size; i++) { if (!isHexNumberChar(start[i])) { return false; @@ -28,7 +28,7 @@ constexpr inline bool isHexNumber(const char *start, const size_t size) noexcept } template -inline void writeString(IoBuffer &buffer, const T &string) { + void writeString(IoBuffer &buffer, const T &string) { buffer.put('"'); for (const char c : string) { switch (c) { @@ -135,19 +135,19 @@ inline std::string_view readKey(IoBuffer &buffer) { return buffer.asString(start, static_cast(i - start)); } -inline constexpr void consumeWhitespace(IoBuffer &buffer) { + constexpr void consumeWhitespace(IoBuffer &buffer) { while (buffer.position() < buffer.size() && isWhitespace(buffer.at(buffer.position()))) { buffer.skip(1); } } template -inline constexpr void assignArray(std::vector &value, const std::vector &result) { + constexpr void assignArray(std::vector &value, const std::vector &result) { value = result; } template -inline constexpr void assignArray(std::array &value, const std::vector &result) { + constexpr void assignArray(std::array &value, const std::vector &result) { if (N != result.size()) { throw ProtocolException("vector -> array size mismatch: source<{}>[{}]={} vs. destination std::array", typeName, result.size(), result, typeName, N); } @@ -156,9 +156,9 @@ inline constexpr void assignArray(std::array &value, const std::vector } } -inline constexpr void skipValue(IoBuffer &buffer); + constexpr void skipValue(IoBuffer &buffer); -inline void skipField(IoBuffer &buffer) { +inline void skipField(IoBuffer &buffer) { consumeWhitespace(buffer); std::ignore = readString(buffer); consumeWhitespace(buffer); @@ -170,7 +170,7 @@ inline void skipField(IoBuffer &buffer) { consumeWhitespace(buffer); } -inline constexpr void skipObject(IoBuffer &buffer) { + constexpr void skipObject(IoBuffer &buffer) { if (buffer.get() != '{') { throw ProtocolException("error: expected {"); } @@ -187,13 +187,13 @@ inline constexpr void skipObject(IoBuffer &buffer) { consumeWhitespace(buffer); } -inline constexpr void skipNumber(IoBuffer &buffer) { + constexpr void skipNumber(IoBuffer &buffer) { while (isNumberChar(buffer.get())) { } buffer.skip(-1); } -inline constexpr void skipArray(IoBuffer &buffer) { + constexpr void skipArray(IoBuffer &buffer) { if (buffer.get() != '[') { throw ProtocolException("error: expected ["); } @@ -211,13 +211,13 @@ inline constexpr void skipArray(IoBuffer &buffer) { consumeWhitespace(buffer); } -inline constexpr void skipValue(IoBuffer &buffer) { + constexpr void skipValue(IoBuffer &buffer) { consumeWhitespace(buffer); const auto firstChar = buffer.at(buffer.position()); if (firstChar == '{') { - json::skipObject(buffer); + skipObject(buffer); } else if (firstChar == '[') { - json::skipArray(buffer); + skipArray(buffer); } else if (firstChar == '"') { std::ignore = readString(buffer); } else if (buffer.size() - buffer.position() > 5 && buffer.asString(buffer.position(), 5) == "false") { @@ -227,17 +227,17 @@ inline constexpr void skipValue(IoBuffer &buffer) { } else if (buffer.size() - buffer.position() > 4 && buffer.asString(buffer.position(), 4) == "null") { buffer.skip(4); } else { // skip number - json::skipNumber(buffer); + skipNumber(buffer); } } } // namespace json template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return 1; } - constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { + static constexpr uint8_t getDataTypeId() { return 1; } + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const END_MARKER &/*value*/) noexcept { using namespace std::string_view_literals; - buffer.put("}"sv); + buffer.put("}"sv); } constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*field*/, const END_MARKER &) { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -246,10 +246,10 @@ struct IoSerialiser { template<> struct IoSerialiser { // catch all template - inline static constexpr uint8_t getDataTypeId() { return 2; } - constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const START_MARKER &/*value*/) noexcept { + static constexpr uint8_t getDataTypeId() { return 2; } + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const START_MARKER &/*value*/) noexcept { using namespace std::string_view_literals; - buffer.put("{"sv); + buffer.put("{"sv); } constexpr static void deserialise(IoBuffer & /*buffer*/, FieldDescription auto const & /*field*/, const START_MARKER & /*value*/) { // do not do anything, as the end marker is of size zero and only the type byte is important @@ -258,8 +258,8 @@ struct IoSerialiser { // catch all template template<> struct IoSerialiser { // because json does not explicitly provide the datatype, all types except nested classes provide data type OTHER - inline static constexpr uint8_t getDataTypeId() { return 0; } - constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const OTHER &) { + static constexpr uint8_t getDataTypeId() { return 0; } + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const OTHER &) { json::skipValue(buffer); } }; @@ -269,7 +269,7 @@ struct FieldHeaderWriter { template constexpr std::size_t static put(IoBuffer &buffer, FieldDescription auto &&field, const DataType &data) { using namespace std::string_view_literals; - constexpr auto WITHOUT = opencmw::IoBuffer::MetaInfo::WITHOUT; + constexpr auto WITHOUT = IoBuffer::MetaInfo::WITHOUT; if constexpr (std::is_same_v) { if (field.fieldName.size() > 0 && field.hierarchyDepth > 0) { buffer.put("\""sv); @@ -280,7 +280,7 @@ struct FieldHeaderWriter { return 0; } if constexpr (std::is_same_v) { - if (buffer.template at(buffer.size() - 2) == ',') { + if (buffer.at(buffer.size() - 2) == ',') { // proceeded by value, remove trailing comma buffer.resize(buffer.size() - 2); } @@ -322,7 +322,7 @@ struct FieldHeaderReader { if (buffer.at(buffer.position()) == '}') { // end marker buffer.skip(1); result.intDataType = IoSerialiser::getDataTypeId(); - result.dataStartPosition = buffer.position() + 1; + result.dataStartPosition = buffer.position(); result.dataEndPosition = std::numeric_limits::max(); // not defined for non-skippable data // set rest of fields return; @@ -355,10 +355,10 @@ struct FieldHeaderReader { template<> struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const bool &value) noexcept { using namespace std::string_view_literals; - buffer.put(value ? "true"sv : "false"sv); + buffer.put(value ? "true"sv : "false"sv); } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const & /*field*/, bool &value) { using namespace std::string_view_literals; @@ -379,7 +379,7 @@ struct IoSerialiser { template struct IoSerialiser { - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) { buffer.reserve_spare(30); // just reserve some spare capacity and expect that all numbers are shorter const auto start = buffer.size(); @@ -432,7 +432,7 @@ struct IoSerialiser { template struct IoSerialiser { // catch all template - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) noexcept { json::writeString(buffer, value); } @@ -444,9 +444,10 @@ struct IoSerialiser { // catch all template template struct IoSerialiser { // todo: arrays of objects - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } - inline constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &values) noexcept { - using MemberType = typename T::value_type; + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &values) noexcept { + using MemberType = T::value_type; buffer.put('['); bool first = true; for (auto value : values) { @@ -460,7 +461,7 @@ struct IoSerialiser { buffer.put(']'); } constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &field, T &value) { - using MemberType = typename T::value_type; + using MemberType = T::value_type; if (buffer.get() != '[') { throw ProtocolException("expected ["); } @@ -473,7 +474,7 @@ struct IoSerialiser { IoSerialiser::deserialise(buffer, field, entry); result.push_back(entry); json::consumeWhitespace(buffer); - const auto next = buffer.template get(); + const auto next = buffer.get(); if (next == ']') { break; } @@ -489,35 +490,35 @@ struct IoSerialiser { template struct IoSerialiser { - using V = typename T::value_type; - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER + using V = T::value_type; + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &value) noexcept { using namespace std::string_view_literals; - buffer.put("{\n"sv); + buffer.put("{\n"sv); std::array dims; for (uint32_t i = 0U; i < T::n_dims_; i++) { dims[i] = static_cast(value.dimensions()[i]); } FieldDescriptionShort memberField{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; memberField.fieldName = "dims"sv; - FieldHeaderWriter::template put(buffer, memberField, dims); + FieldHeaderWriter::put(buffer, memberField, dims); memberField.fieldName = "values"sv; - FieldHeaderWriter::template put(buffer, memberField, value.elements()); + FieldHeaderWriter::put(buffer, memberField, value.elements()); buffer.resize(buffer.size() - 2); // remove trailing comma - buffer.put("\n}\n"sv); + buffer.put("\n}\n"sv); } static void deserialise(IoBuffer &buffer, FieldDescription auto const &field, T &value) { using namespace std::string_view_literals; DeserialiserInfo info{}; - FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = ExternalModifier::UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; - FieldHeaderReader::template get(buffer, info, fieldHeader); + FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; + FieldHeaderReader::get(buffer, info, fieldHeader); if (fieldHeader.intDataType != IoSerialiser>::getDataTypeId() || fieldHeader.fieldName != "dims"sv) { throw ProtocolException("expected multi-array dimensions"); } std::array dimWire; IoSerialiser>::deserialise(buffer, fieldHeader, dimWire); for (auto i = 0U; i < T::n_dims_; i++) { - value.dimensions()[i] = static_cast(dimWire[i]); + value.dimensions()[i] = static_cast(dimWire[i]); } value.element_count() = value.dimensions()[T::n_dims_ - 1]; value.stride(T::n_dims_ - 1) = 1; @@ -527,12 +528,12 @@ struct IoSerialiser { value.stride(i - 1) = value.stride(i) * value.dimensions()[i]; value.offset(i - 1) = 0; } - FieldHeaderReader::template get(buffer, info, fieldHeader); + FieldHeaderReader::get(buffer, info, fieldHeader); if (fieldHeader.intDataType != IoSerialiser>::getDataTypeId() || fieldHeader.fieldName != "values"sv) { throw ProtocolException("expected multi-array values"); } IoSerialiser>::deserialise(buffer, fieldHeader, value.elements()); - FieldHeaderReader::template get(buffer, info, fieldHeader); + FieldHeaderReader::get(buffer, info, fieldHeader); if (fieldHeader.intDataType != IoSerialiser::getDataTypeId()) { throw ProtocolException("expected end marker"); } @@ -542,8 +543,9 @@ struct IoSerialiser { template struct IoSerialiser> { // todo: arrays of objects - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } - inline constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const std::set &values) noexcept { + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const std::set &values) noexcept { buffer.put('['); bool first = true; for (auto &value : values) { @@ -569,7 +571,7 @@ struct IoSerialiser> { IoSerialiser::deserialise(buffer, field, entry); value.insert(entry); json::consumeWhitespace(buffer); - const auto next = buffer.template get(); + const auto next = buffer.get(); if (next == ']') { break; } @@ -583,30 +585,42 @@ struct IoSerialiser> { }; template requires(is_stringlike) struct IoSerialiser { - using K = typename T::key_type; - using V = typename T::mapped_type; - inline static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER + using K = T::key_type; + using V = T::mapped_type; + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } // because the object is serialised as a subobject, we have to emmit START_MARKER constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &field, const T &value) { using namespace std::string_view_literals; - buffer.put("{\n"sv); + buffer.put("{\n"sv); FieldDescriptionShort memberField{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; for (auto &entry : value) { memberField.fieldName = entry.first; - FieldHeaderWriter::template put(buffer, memberField, entry.second); + FieldHeaderWriter::put(buffer, memberField, entry.second); } buffer.resize(buffer.size() - 2); // remove trailing comma - buffer.put("\n}\n"sv); + buffer.put("\n}\n"sv); } static void deserialise(IoBuffer &buffer, FieldDescription auto const &field, T &value) { using namespace std::string_view_literals; DeserialiserInfo info{}; - FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = ExternalModifier::UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; - for (FieldHeaderReader::template get(buffer, info, fieldHeader); fieldHeader.intDataType != IoSerialiser::getDataTypeId(); FieldHeaderReader::get(buffer, info, fieldHeader)) { + FieldDescriptionLong fieldHeader{ .headerStart = 0, .dataStartPosition = 0, .dataEndPosition = 0, .subfields = 0, .fieldName = ""sv, .unit = ""sv, .description = ""sv, .modifier = UNKNOWN, .intDataType = IoSerialiser::getDataTypeId(), .hierarchyDepth = static_cast(field.hierarchyDepth + 1U) }; + for (FieldHeaderReader::get(buffer, info, fieldHeader); fieldHeader.intDataType != IoSerialiser::getDataTypeId(); FieldHeaderReader::get(buffer, info, fieldHeader)) { auto fieldValue = value[std::string(fieldHeader.fieldName)]; IoSerialiser::deserialise(buffer, fieldHeader, fieldValue); } } }; + +template +struct IoSerialiser { + static constexpr uint8_t getDataTypeId() { return IoSerialiser::getDataTypeId(); } + constexpr static void serialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, const T &value) noexcept { + opencmw::serialise(buffer, value); + } + constexpr static void deserialise(IoBuffer &buffer, FieldDescription auto const &/*field*/, T &value) { + opencmw::deserialise(buffer, value); // TODO: refactor API to correctly propagate ProtocolCheck and Deserialiser Info + } +}; + } // namespace opencmw #endif // OPENCMW_JSONSERIALISER_H diff --git a/src/serialiser/test/IoSerialiserJson_tests.cpp b/src/serialiser/test/IoSerialiserJson_tests.cpp index 4bed5887..ae2784f9 100644 --- a/src/serialiser/test/IoSerialiserJson_tests.cpp +++ b/src/serialiser/test/IoSerialiserJson_tests.cpp @@ -128,11 +128,12 @@ TEST_CASE("JsonDeserialisationMissingField", "[JsonSerialiser]") { opencmw::debug::resetStats(); { opencmw::IoBuffer buffer; - buffer.put(R"({ "float1": 2.3, "superfluousField": { "p":12 , "a":null,"x" : false, "q": [ "a", "s"], "z": [true , false ] }, "test": { "intArray" : [ 1,2, 3], "val1":13.37e2, "val2":"bar"}, "int1": 42})"sv); + buffer.put(R"({ "float1": 2.3, "superfluousField": { "p":12 , "a":null,"x" : false, "q": [ "a", "s"], "z": [true , false ] }, "test": { "intArray" : [ 1,2, 3], "val1":13.37e2, "val2":"bar"}, "int1": 42} )"sv); std::cout << "Prepared json data: " << buffer.asString() << std::endl; Simple foo; auto result = opencmw::deserialise(buffer, foo); std::print(std::cout, "deserialisation finished: {}\n", result); + REQUIRE(buffer.position() == 189UZ); REQUIRE(foo.test.get()->val1 == 1337.0); REQUIRE(foo.test.get()->val2 == "bar"); REQUIRE(foo.test.get()->intArray == std::vector{ 1, 2, 3 }); From 148d345c8c900087d0c24e5a7028bed596435fb2 Mon Sep 17 00:00:00 2001 From: Alexander Krimm Date: Wed, 19 Jun 2024 18:34:56 +0200 Subject: [PATCH 7/7] CmwLightClient: Implement rda3 compatible client Also implements a lightweight rda3 directory client Signed-off-by: Alexander Krimm --- cmake/DependenciesNative.cmake | 12 +- cmake/patches/cpr-disable-std-fs-test.diff | 17 + src/client/CMakeLists.txt | 1 + src/client/include/CmwLightClient.hpp | 958 ++++++++++++++++++++ src/client/include/DirectoryLightClient.hpp | 190 ++++ src/client/test/CMakeLists.txt | 10 + src/client/test/CmwLightTest.cpp | 282 ++++++ src/core/include/Topic.hpp | 10 +- src/zmq/include/zmq/ZmqUtils.hpp | 10 + 9 files changed, 1482 insertions(+), 8 deletions(-) create mode 100644 cmake/patches/cpr-disable-std-fs-test.diff create mode 100644 src/client/include/CmwLightClient.hpp create mode 100644 src/client/include/DirectoryLightClient.hpp create mode 100644 src/client/test/CmwLightTest.cpp diff --git a/cmake/DependenciesNative.cmake b/cmake/DependenciesNative.cmake index 50ab56ea..e76f1533 100644 --- a/cmake/DependenciesNative.cmake +++ b/cmake/DependenciesNative.cmake @@ -106,13 +106,15 @@ FetchContent_Declare( ) set(HTTPLIB_USE_ZSTD_IF_AVAILABLE OFF CACHE BOOL "Disable zstd for httplib") -# zlib: optional httplib dependency +set(CPR_USE_SYSTEM_CURL ON CACHE BOOL "Use system libcurl in CPR" FORCE) +set(cpr_patch_command ) FetchContent_Declare( - zlib - GIT_REPOSITORY https://github.com/madler/zlib.git - GIT_TAG v1.3.12 + cpr + GIT_REPOSITORY https://github.com/libcpr/cpr.git + GIT_TAG 1.14.1 + PATCH_COMMAND git apply --ignore-space-change --ignore-whitespace "${CMAKE_CURRENT_LIST_DIR}/patches/cpr-disable-std-fs-test.diff" || true ) -FetchContent_MakeAvailable(cpp-httplib zeromq) +FetchContent_MakeAvailable(cpp-httplib zeromq cpr) list(APPEND CMAKE_MODULE_PATH ${catch2_SOURCE_DIR}/contrib) # replace contrib by extras for catch2 v3.x.x diff --git a/cmake/patches/cpr-disable-std-fs-test.diff b/cmake/patches/cpr-disable-std-fs-test.diff new file mode 100644 index 00000000..9a7bc5db --- /dev/null +++ b/cmake/patches/cpr-disable-std-fs-test.diff @@ -0,0 +1,17 @@ +diff --git a/cmake/std_fs_support_test.cpp b/cmake/std_fs_support_test.cpp +index 44bac77..237c8ce 100644 +--- a/cmake/std_fs_support_test.cpp ++++ b/cmake/std_fs_support_test.cpp +@@ -1,11 +1 @@ +-#if __has_include() +-#include +-namespace fs = std::filesystem; +-#else +-#include +-namespace fs = std::experimental::filesystem; +-#endif +- +-int main() { +- auto cwd = fs::current_path(); +-} ++int main() {} diff --git a/src/client/CMakeLists.txt b/src/client/CMakeLists.txt index 3ffb281a..c9a44d2c 100644 --- a/src/client/CMakeLists.txt +++ b/src/client/CMakeLists.txt @@ -17,6 +17,7 @@ if(NOT EMSCRIPTEN) serialiser majordomo rest + cpr::cpr zmq) else() target_link_libraries( diff --git a/src/client/include/CmwLightClient.hpp b/src/client/include/CmwLightClient.hpp new file mode 100644 index 00000000..e2987951 --- /dev/null +++ b/src/client/include/CmwLightClient.hpp @@ -0,0 +1,958 @@ +#ifndef OPENCMW_CPP_CMWLIGHTCLIENT_HPP +#define OPENCMW_CPP_CMWLIGHTCLIENT_HPP + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +// TODO: +// - subscription filters: allow filters with different types by prefixing "int:" etc to the string map value +// - see if simplification is possible: now requests are first queued in a ring buffer and then dispatched to the corresponding client via local zeromq before the actual zeromq request is made +// - speed up initial setup by not having to wait multiple houskeeping timeouts + +namespace opencmw::client::cmwlight { + +struct Request { + URI<> uri; + std::function callback; + timePoint timestamp_received = std::chrono::system_clock::now(); +}; + +struct Subscription { + URI<> uri; + std::function callback; + timePoint timestamp_received = std::chrono::system_clock::now(); + std::size_t reqId; +}; + +struct CmwLightHeaderOptions { + int64_t b{}; // SOURCE_ID + std::map e; // SESSION_BODY + // can potentially contain more and arbitrary data + // accessors to make code more readable + int64_t &sourceId() { return b; } + std::map &sessionBody() { return e; } +}; + +struct CmwLightHeader { + int8_t x_2{}; // REQ_TYPE_TAG + int64_t x_0{}; // ID_TAG + std::string x_1; // DEVICE_NAME + std::string f; // PROPERTY_NAME + int8_t x_7{}; // UPDATE_TYPE + std::string d; // SESSION_ID + std::unique_ptr x_3; + // accessors to make code more readable + int8_t &requestType() { return x_2; } + int64_t &id() { return x_0; } + std::string &device() { return x_1; } + std::string &property() { return f; } + int8_t &updateType() { return x_7; } + std::string &sessionId() { return d; } + std::unique_ptr &options() { return x_3; } +}; + +struct CmwLightConnectBody { + std::string x_9; + std::string &clientInfo() { return x_9; } +}; + +struct CmwLightRequestContext { + std::string x_8; // SELECTOR + std::map> c; // FILTERS + std::map> x; // DATA + // accessors to make code more readable + std::string &selector() { return x_8; }; + std::map> &filters() { return c; } + std::map> &data() { return x; } +}; + +struct CmwLightDataContext { + std::string x_4; // CYCLE_NAME + int64_t x_6; // CYCLE_STAMP + int64_t x_5; // ACQ_STAMP + std::map x; // DATA // todo: support arbitrary filter data std::map> + // accessors to make code more readable + std::string &cycleName() { return x_4; } + long &cycleStamp() { return x_6; } + long &acqStamp() { return x_5; } + std::map &data() { return x; } +}; + +} // namespace opencmw::client::cmwlight +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightHeaderOptions, b, e) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightHeader, x_2, x_0, x_1, f, x_7, d, x_3) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightConnectBody, x_9) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightRequestContext, x_8, c, x) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::CmwLightDataContext, x_4, x_6, x_5, x) + +namespace opencmw::client::cmwlight { +namespace detail { +/** + * Sent as the first frame of an RDA3 message determining the type of message + */ +enum class MessageType : char { + SERVER_CONNECT_ACK = 0x01, + SERVER_REP = 0x02, + SERVER_HB = 0x03, + CLIENT_CONNECT = 0x20, + CLIENT_REQ = 0x21, + CLIENT_HB = 0x22 +}; + +/** + * Frame Types in the descriptor (Last frame of a message containing the type of each sub message) + */ +enum class FrameType : char { + HEADER = 0, + BODY = 1, + BODY_DATA_CONTEXT = 2, + BODY_REQUEST_CONTEXT = 3, + BODY_EXCEPTION = 4 +}; + +/** + * request type used in request header REQ_TYPE_TAG + */ +enum class RequestType : char { + GET = 0, + SET = 1, + CONNECT = 2, + REPLY = 3, + EXCEPTION = 4, + SUBSCRIBE = 5, + UNSUBSCRIBE = 6, + NOTIFICATION_DATA = 7, + NOTIFICATION_EXC = 8, + SUBSCRIBE_EXCEPTION = 9, + EVENT = 10, + SESSION_CONFIRM = 11 +}; + +/** + * UpdateType + */ +enum class UpdateType : char { + NORMAL = 0, + FIRST_UPDATE = 1, + IMMEDIATE_UPDATE = 2 +}; + +inline std::string getHostName() { + std:: string hostname; + hostname.resize(255); + if (const int result = gethostname(hostname.data(), hostname.capacity()); !result) { + hostname = "localhost"; + } else { + hostname.resize(strnlen(hostname.data(), hostname.size())); + hostname.shrink_to_fit(); + } + return hostname; +} + +inline std::string getIdentity() { + std::string hostname = getHostName(); + static int CONNECTION_ID_GENERATOR = 1; + static int channelIdGenerator = 1; // todo: make this per connection + return std::format("{}/{}/{}/{}", hostname, getpid(), ++CONNECTION_ID_GENERATOR, ++channelIdGenerator); // N.B. this scheme is parsed/enforced by CMW +} + +inline std::string percentEncode(const std::string_view &str) { + std::string result; + result.reserve(str.size() * 3); // this is the upper bound if all characters have to be percent encoded, avoids reallocation + for (const char c : str) { + if (isalnum(c)) { + result += c; + } else { + result += '%' + std::format("{:02X}", static_cast(c)); + } + } + return result; +} + +inline std::string createClientInfo() { + const std::string hostname = getHostName(); + const char *usernamePtr = getlogin(); + const std::string username = usernamePtr ? usernamePtr : "unknown"; + const std::string processName = program_invocation_short_name; + const std::string language = "cpp"; + const long startTime = std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); + const int pid = getpid(); + const std::string version = "2026.2.1"; + std::size_t applicationIdSize = 26 + processName.size() + username.size() + hostname.size() + std::to_string(pid).size(); + const std::string applicationId = std::format("app={};ver={};uid={};host={};pid={};", percentEncode(processName), percentEncode(version), percentEncode(username), percentEncode(hostname), std::to_string(pid)); + return "9" // number of fields + + std::format("#Address:#string#{}#tcp:%2F%2F{}:0", hostname.size() + 8, percentEncode(hostname)) + + std::format("#ApplicationId:#string#{}#{}", applicationIdSize, applicationId) + + std::format("#UserName:#string#{}#{}", username.size(), percentEncode(username)) + + std::format("#ProcessName:#string#{}#{}", processName.size(), percentEncode(processName)) + + std::format("#Language:#string#{}#{}", language.size(), percentEncode(language)) + + std::format("#StartTime:#long#{}", startTime) + + std::format("#Name:#string#{}#{}", processName.size(), percentEncode(processName)) + + std::format("#Pid:#int#{}", pid) + + std::format("#Version:#string#{}#{}", version.size(), percentEncode(version)); +} + +inline std::string createClientId() { + const char *usernamePtr = getlogin(); + const std::string hostname = getHostName(); + const std::string username = usernamePtr ? usernamePtr : "unknown"; + const std::string processName = program_invocation_short_name; + const int pid = getpid(); + const std::string version = "2026.2.1"; + const auto startTime = std::chrono::system_clock::now(); + return std::format("RemoteHostInfoImpl[name={}; userName={}; appId=[app={};ver={};uid={};host={};pid={};]; process={}; pid={}; address=tcp://{}:0; startTime={:%F %R%z}; connectionTime=About ago; version={}; language=CPP]1", processName, username, processName, version, username, hostname, pid, processName, pid, hostname, startTime, version); +} + +struct PendingRequest { + enum class RequestState { + INITIALIZED, + WAITING, + FINISHED + }; + std::string reqId; + IoBuffer data{}; + RequestType requestType{ RequestType::GET }; + RequestState state{ RequestState::INITIALIZED }; + std::string uri; +}; + +struct OpenSubscription { + enum class SubscriptionState { + INITIALIZED, + SUBSCRIBING, + SUBSCRIBED, + UNSUBSCRIBING, + UNSUBSCRIBED + }; + std::chrono::milliseconds backOff = 20ms; + long updateId{}; + long reqId = 0L; + long replyId{}; + SubscriptionState state = SubscriptionState::SUBSCRIBING; + std::chrono::system_clock::time_point nextTry; + std::string uri; +}; + +struct Connection { + enum class ConnectionState { + DISCONNECTED, + NS_LOOKUP, + CONNECTING1, + CONNECTING2, + CONNECTED, + }; + std::string _deviceName; + std::string _authority; + zmq::Socket _socket; + ConnectionState _connectionState = ConnectionState::DISCONNECTED; + timePoint _nextReconnectAttemptTimeStamp = std::chrono::system_clock::now(); + timePoint _lastHeartbeatReceived = std::chrono::system_clock::now(); + timePoint _lastHeartBeatSent = std::chrono::system_clock::now(); + std::chrono::milliseconds _backoff = 20ms; // implements exponential back-off to get + std::vector _frames{}; // currently received frames; will be accumulated until the message is complete + std::map _subscriptions; // all subscriptions requested for (un)subscribe + int64_t _subscriptionIdGenerator = 1; + int64_t _requestIdGenerator = 1; + std::map _pendingRequests; + + Connection(const zmq::Context &context, const std::string_view authority, const int zmq_dealer_type) : _authority{ authority }, _socket{ context, zmq_dealer_type } { + zmq::initializeSocket(_socket).assertSuccess(); + } +}; + +static void send(const zmq::Socket &socket, const int param, std::string_view errorMsg, auto &&data) { + if (zmq::MessageFrame connectFrame{ FWD(data) }; !connectFrame.send(socket, param).isValid()) { + throw std::runtime_error(errorMsg.data()); + } +} + +static std::string descriptorToString(auto... descriptor) { + std::string result{}; + result.reserve(sizeof...(descriptor)); + ((result.push_back(static_cast(descriptor))), ...); + return result; +} + +static IoBuffer serialiseCmwLight(auto &requestType) { + IoBuffer buffer{}; + opencmw::serialise(buffer, requestType); + buffer.reset(); + return buffer; +} + +inline void sendConnectRequest(Connection &con) { + using namespace std::string_view_literals; + detail::send(con._socket, ZMQ_SNDMORE, "error sending get frame"sv, static_cast(MessageType::CLIENT_REQ)); + CmwLightHeader header; + header.requestType() = static_cast(detail::RequestType::CONNECT); + header.id() = con._requestIdGenerator++; + header.options() = std::make_unique(); + send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, serialiseCmwLight(header)); // send message header + CmwLightConnectBody connectBody; + connectBody.clientInfo() = createClientInfo(); + send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, serialiseCmwLight(connectBody)); // send message header + using enum detail::FrameType; + send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY)); +} +} // namespace detail + +class CMWLightClientBase { +public: + virtual ~CMWLightClientBase() = default; + virtual bool receive(mdp::Message &message) = 0; + virtual timePoint housekeeping(const timePoint &now) = 0; + virtual void get(const URI<> &, std::string_view) = 0; + virtual void set(const URI<> &, std::string_view, const std::span &) = 0; + virtual void subscribe(const URI<> &, std::string_view) = 0; + virtual void unsubscribe(const URI<> &, std::string_view) = 0; +}; + +class CMWLightClient : public CMWLightClientBase { + using timeUnit = std::chrono::milliseconds; + const timeUnit _clientTimeout; + const zmq::Context &_context; + DirectoryLightClient &_directoryClient; + const std::string _clientId; + const std::string _sourceName; + std::vector _connections; + std::vector &_pollItems; + constexpr static auto HEARTBEAT_INTERVAL = 2000ms; + +public: + explicit CMWLightClient(const zmq::Context &context, + std::vector &pollItems, + DirectoryLightClient &directoryClient, + const timeUnit timeout = 1s, + std::string clientId = "") + : _clientTimeout(timeout), _context(context), _directoryClient(directoryClient), _clientId(std::move(clientId)), _sourceName(std::format("CMWLightClient(clientId: {})", _clientId)), _pollItems(pollItems) {} + + void connect(detail::Connection &con) const { + using enum detail::Connection::ConnectionState; + using namespace std::string_view_literals; + // todo: for now we expect rda3tcp://host:port, but this should allow be rda3:///devicename which will be looked up on the cmw directory server + auto endpoint = std::format("tcp://{}", con._authority); + std::string id = detail::getIdentity(); + if (!zmq::invoke(zmq_setsockopt, con._socket, ZMQ_IDENTITY, id.data(), id.size()).isValid()) { // hostname/process/id/channel -- the server verifies this + throw std::runtime_error("failed set socket identity"); + } + if (zmq::invoke(zmq_connect, con._socket, endpoint).isValid()) { + _pollItems.push_back({ .socket = con._socket.zmq_ptr, .fd = 0, .events = ZMQ_POLLIN, .revents = 0 }); + } + // send rda3 connect message + detail::send(con._socket, ZMQ_SNDMORE, "error sending connect frame"sv, static_cast(detail::MessageType::CLIENT_CONNECT)); + detail::send(con._socket, 0, "error sending connect frame"sv, "1.0.0"); + con._connectionState = CONNECTING1; + } + + detail::Connection &findConnection(const URI<> &uri) { + const std::string uriPath = uri.path().value(); + const auto deviceEndPos = uriPath.find('/', 1); + const std::string_view deviceName = std::string_view{uriPath}.substr(1, (deviceEndPos == std::string_view::npos ? uriPath.size() : deviceEndPos) - 1); + const auto con = std::ranges::find_if(_connections, [deviceName](const detail::Connection &c) { return c._deviceName == deviceName; }); + if (con == _connections.end()) { + auto newCon = detail::Connection(_context, uri.authority().value(), ZMQ_DEALER); + newCon._deviceName = deviceName; + newCon._authority = uri.authority().value_or(""); + _connections.push_back(std::move(newCon)); + return _connections.back(); + } + return *con; + } + + void get(const URI<> &uri, const std::string_view req_id) override { + using namespace std::string_view_literals; + using enum detail::FrameType; + auto &con = findConnection(uri); // send message header + detail::PendingRequest req{}; + req.reqId = req_id; + req.requestType = detail::RequestType::GET; + req.state = detail::PendingRequest::RequestState::INITIALIZED; + req.uri = uri.str(); + con._pendingRequests.insert({ std::string(req_id), std::move(req) }); + } + + void set(const URI<> &uri, const std::string_view req_id, const std::span &request) override { + using namespace std::string_view_literals; + using enum detail::FrameType; + auto &con = findConnection(uri); // send message header + detail::PendingRequest req{}; + req.reqId = req_id; + req.requestType = detail::RequestType::SET; + req.data = IoBuffer{ request.data(), request.size() }; + req.state = detail::PendingRequest::RequestState::INITIALIZED; + req.uri = uri.str(); + con._pendingRequests.insert({ std::string(req_id), std::move(req) }); + } + + void subscribe(const URI<> &uri, const std::string_view req_id) override { + using namespace std::string_view_literals; + using enum detail::FrameType; + auto &con = findConnection(uri); + detail::OpenSubscription sub{}; + sub.state = detail::OpenSubscription::SubscriptionState::INITIALIZED; + sub.uri = uri.str(); + std::string req_id_string{ req_id }; + char *req_id_end = req_id_string.data() + req_id_string.size(); + sub.reqId = strtol(req_id_string.data(), &req_id_end, 10); + con._subscriptions.insert({ std::string(req_id), std::move(sub) }); + } + + void unsubscribe(const URI<> &uri, const std::string_view req_id) override { + using namespace std::string_view_literals; + auto &con = findConnection(uri); + con._subscriptions[std::string{ req_id }].state = detail::OpenSubscription::SubscriptionState::UNSUBSCRIBING; + CmwLightHeader header; + header.requestType() = static_cast(detail::RequestType::UNSUBSCRIBE); + std::string reqIdString{ req_id }; + char *end = reqIdString.data() + req_id.size(); + header.id() = std::strtol(req_id.data(), &end, 10); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, detail::serialiseCmwLight(header)); // send message header + CmwLightRequestContext ctx; + // send requestContext + using enum detail::FrameType; + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER)); + } + + bool disconnect(detail::Connection &con) const { +#if not defined(__EMSCRIPTEN__) and (not defined(__clang__) or (__clang_major__ >= 16)) + const auto remove = std::ranges::remove_if(_pollItems, [&con](const zmq_pollitem_t &pollItem) { return pollItem.socket == con._socket.zmq_ptr; }); + _pollItems.erase(remove.begin(), remove.end()); +#else + const auto remove = std::remove_if(_pollItems.begin(), _pollItems.end(), [&con](zmq_pollitem_t &pollItem) { return pollItem.socket == con._socket.zmq_ptr; }); + _pollItems.erase(remove, _pollItems.end()); +#endif + con._connectionState = detail::Connection::ConnectionState::DISCONNECTED; + return true; + } + + static bool handleServerReply(mdp::Message &output, detail::Connection &con, const auto currentTime) { + using namespace std::string_view_literals; + if (con._frames.size() < 2 || con._frames.back().data().size() != con._frames.size() - 2 || static_cast(con._frames.back().data()[0]) != detail::FrameType::HEADER) { + throw std::runtime_error(std::format("received malformed response: wrong number of frames({}) or mismatch with frame descriptor({})", con._frames.size(), con._frames.back().size())); + } + // deserialise header frames[1] + IoBuffer data(con._frames[1].data().data(), con._frames[1].data().size()); + DeserialiserInfo info = checkHeaderInfo(data, DeserialiserInfo{}, ProtocolCheck::LENIENT); + CmwLightHeader header; + auto result = opencmw::deserialise(data, header); + + if (con._connectionState == detail::Connection::ConnectionState::CONNECTING2) { + if (header.requestType() == static_cast(detail::RequestType::REPLY)) { + con._connectionState = detail::Connection::ConnectionState::CONNECTED; + con._lastHeartbeatReceived = currentTime; + return true; + } else { + throw std::runtime_error("expected connection reply but got different message"); + } + } + + using enum detail::RequestType; + switch (detail::RequestType{ static_cast(header.requestType()) }) { + case REPLY: { + auto request = con._pendingRequests[std::format("{}", header.id())]; + // con._pendingRequests.erase(header.id()); + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Final; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + char *end = &request.reqId.back(); + output.id = std::strtoul(request.reqId.data(), &end, 10); /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", header.id()); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{request.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{ con._frames[2].data().data(), con._frames[2].size() }; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = ""; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") + // output.rbac; ///IoBuffer < optional RBAC meta-info -- may contain token, role, signed message hash (implementation dependent) + return true; + } + case EXCEPTION: { + auto request = con._pendingRequests[std::format("{}", header.id())]; + // con._pendingRequests.erase(header.id()); + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Final; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + char *end = &request.reqId.back(); + output.id = std::strtoul(request.reqId.data(), &end, 10); /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", header.id()); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{request.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{}; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = std::string{con._frames[2].data().data(), con._frames[2].size()}; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") + return true; + } + case SUBSCRIBE: { + auto &sub = con._subscriptions[std::format("{}", header.id())]; + // todo: handle nonexisting subscription + sub.replyId = header.options()->sourceId(); + sub.state = detail::OpenSubscription::SubscriptionState::SUBSCRIBED; + sub.backOff = 20ms; // reset back-off + return false; + } + case UNSUBSCRIBE: { + auto subscriptionForUnsub = con._subscriptions.find(std::format("{}", header.id())); + con._subscriptions.erase(subscriptionForUnsub); + return false; + } + case NOTIFICATION_DATA: { + if (auto sub = std::ranges::find_if(con._subscriptions, [&header](auto &pair) { return pair.second.replyId == header.id(); });sub == con._subscriptions.end()) { + // std::println("received unexpected subscription for replyId: {}", header.id()); + // todo: add a temporary subscription to properly unsubscribe this untracked subscription? + return false; + } else { + auto subscriptionForNotification = sub->second; // con._subscriptions[replyId]; + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Notify; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + output.id = static_cast(sub->second.replyId); /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", subscriptionForNotification.reqId); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{subscriptionForNotification.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{ con._frames[2].data().data(), con._frames[2].size() }; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = ""; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") + return true; + } + } + case NOTIFICATION_EXC: { + std::string replyId; + if (auto subIt = con._subscriptions.find(replyId); subIt == con._subscriptions.end()) { + return false; + } else { + auto subscriptionForNotifyExc = subIt->second; + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Notify; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + output.id = 0; /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", subscriptionForNotifyExc.reqId); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{subscriptionForNotifyExc.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{}; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = std::string{con._frames[2].data().data(), con._frames[2].size()}; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") // TODO: parse error message as cmwlight with fields "ContextAcqStamp", "ContextCycleStamp", "Message", "Type" + return true; + } + } + case SUBSCRIBE_EXCEPTION: { + auto subForSubExc = con._subscriptions[std::format("{}", header.id())]; + subForSubExc.state = detail::OpenSubscription::SubscriptionState::UNSUBSCRIBED; + subForSubExc.nextTry = currentTime + subForSubExc.backOff; + subForSubExc.backOff *= 2; + // exception during subscription, retrying + output.arrivalTime = std::chrono::system_clock::now(); /// timePoint < UTC time when the message was sent/received by the client + output.command = mdp::Command::Notify; /// Command < command type (GET, SET, SUBSCRIBE, UNSUBSCRIBE, PARTIAL, FINAL, NOTIFY, READY, DISCONNECT, HEARTBEAT) + output.id = static_cast(subForSubExc.reqId); /// std::size_t + output.protocolName = "RDA3"; /// std::string < unique protocol name including version (e.g. 'MDPC03' or 'MDPW03') + std::string clientRequestID = std::format("{}", subForSubExc.reqId); + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.clientRequestID = IoBuffer{clientRequestID.data(), clientRequestID.size()}; /// IoBuffer < stateful: worker mirrors clientRequestID; stateless: worker generates unique increasing IDs (to detect packet loss) + output.topic = URI{subForSubExc.uri}; /// URI < URI containing at least and optionally parameters + output.serviceName = output.topic.path().value_or("/"); /// std::string < service endpoint name (normally the URI path only), or client source ID (for broker <-> worker messages) + output.data = IoBuffer{}; /// IoBuffer < request/reply body -- opaque binary, e.g. YaS-, CmwLight-, JSON-, or HTML-based + output.error = std::string{con._frames[2].data().data(), con._frames[2].size()}; /// std::string < UTF-8 strings containing error code and/or stack-trace (e.g. "404 Not Found") // TODO: parse error message as cmwlight with fields "ContextAcqStamp", "ContextCycleStamp", "Message", "Type" + return true; + } + case SESSION_CONFIRM: { + return false; + } + // These request types should never be returned by the server or are unsupported by this implementation + case GET: + case SET: + case CONNECT: + case EVENT: + default: + // std::println("unsupported message: {}", header.requestType()); + return false; + } + } + + static bool handleMessage(mdp::Message &output, detail::Connection &con) { + assert(!con._frames.empty() && "this function can only be ever called with at least one frame"); + const auto currentTime = std::chrono::system_clock::now(); + using enum detail::MessageType; + using enum detail::Connection::ConnectionState; + switch (static_cast(con._frames[0].data().at(0))) { + case SERVER_CONNECT_ACK: + if (con._connectionState == CONNECTING1) { + if (con._frames.size() < 2 || con._frames[1].data().empty()) { + throw std::runtime_error("server connect does not contain required version info"); + } + // verifyVersion(con._frames[1].data()); // todo: implement checking rda3 protocol version + con._connectionState = CONNECTING2; // proceed to step 2 by sending the CLIENT_REQ, REQ_TYPE=CONNECT message + sendConnectRequest(con); + con._lastHeartbeatReceived = currentTime; + con._backoff = 20ms; // reset back-off time + } else { + throw std::runtime_error("received unsolicited SERVER_CONNECT_ACK"); + } + break; + case SERVER_HB: + if (con._connectionState != CONNECTED && con._connectionState != CONNECTING2) { + // std::println("received a heart-beat message on an unconnected connection!"); + return false; + } + con._lastHeartbeatReceived = currentTime; + break; + case SERVER_REP: + con._lastHeartbeatReceived = currentTime; + return handleServerReply(output, con, currentTime); + case CLIENT_CONNECT: + case CLIENT_REQ: + case CLIENT_HB: + default: + throw std::runtime_error("Unexpected client message type received from server"); + } + return false; + } + + bool receive(mdp::Message &output) override { + for (auto &con : _connections) { + while (true) { + zmq::MessageFrame frame; + if (const auto byteCountResultId = frame.receive(con._socket, ZMQ_DONTWAIT); !byteCountResultId.isValid() || byteCountResultId.value() < 1) { + break; + } + con._frames.push_back(std::move(frame)); + int64_t more; + size_t moreSize = sizeof(more); + if (!zmq::invoke(zmq_getsockopt, con._socket, ZMQ_RCVMORE, &more, &moreSize)) { + throw std::runtime_error("error checking rcvmore"); + } else if (more == 0) { // the message is complete + const bool received = handleMessage(output, con); + con._frames.clear(); + if (received) { + return true; + } + } + } + } + return false; + } + + // method to be called in regular time intervals to send and verify heartbeats + timePoint housekeeping(const timePoint &now) override { + using ConnectionState = detail::Connection::ConnectionState; + using RequestState = detail::PendingRequest::RequestState; + using namespace std::literals; + using enum detail::OpenSubscription::SubscriptionState; + using enum detail::FrameType; + // handle connection state + for (auto &con : _connections) { + switch (con._connectionState) { + case ConnectionState::DISCONNECTED: + if (con._nextReconnectAttemptTimeStamp <= now) { + if (con._authority.empty()) { + con._connectionState = ConnectionState::NS_LOOKUP; + } else { + connect(con); + } + } + break; + case ConnectionState::NS_LOOKUP: { + auto deviceName = con._deviceName; + if (auto address = _directoryClient.lookup(deviceName); address.has_value()) { + con._authority = address.value(); + connect(con); + } + break; + } + case ConnectionState::CONNECTING1: + case ConnectionState::CONNECTING2: { + if (con._nextReconnectAttemptTimeStamp + _clientTimeout < now) { + // abort this connection attempt and start a new one + } + break; + } + case ConnectionState::CONNECTED: + for (auto &req : con._pendingRequests | std::views::values) { + using enum detail::RequestType; + if (req.state == RequestState::INITIALIZED) { + if (req.requestType == GET && !req.reqId.empty()) { + detail::send(con._socket, ZMQ_SNDMORE, "error sending get frame"sv, static_cast(detail::MessageType::CLIENT_REQ) ); + CmwLightHeader msg; + msg.requestType() = static_cast(GET); + char *reqIdEnd = req.reqId.data() + req.reqId.size(); + msg.id() = std::strtol(req.reqId.data(), &reqIdEnd, 10); + msg.sessionId() = detail::createClientId(); + URI uri{ req.uri }; + msg.device() = uri.path()->substr(1, uri.path()->find('/', 1) - 1); + msg.property() = uri.path()->substr(uri.path()->find('/', 1) + 1); + msg.options() = std::make_unique(); + msg.updateType() = static_cast(detail::UpdateType::NORMAL); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, detail::serialiseCmwLight(msg)); // send message header + if (auto params = uri.queryParamMap(); params.contains("ctx")) { + CmwLightRequestContext ctx; + for (auto &[key, value] : params) { + if (key == "ctx") { + ctx.selector() = value.value_or("FAIR.SELECTOR.ALL"); + } else { + ctx.filters().insert({key, value.value_or("")}); + } + } + IoBuffer buffer{}; + serialise(buffer, ctx); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send context frame"sv, std::move(buffer)); // send requestContext + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY_REQUEST_CONTEXT)); + } else { + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER)); + } + req.state = RequestState::WAITING; + } else if (req.requestType == SET) { + detail::send(con._socket, ZMQ_SNDMORE, "error sending get frame"sv, static_cast(detail::MessageType::CLIENT_REQ)); + CmwLightHeader msg; + msg.requestType() = static_cast(detail::RequestType::GET); + char *reqIdEnd = req.reqId.data() + req.reqId.size(); + msg.id() = std::strtol(req.reqId.data(), &reqIdEnd, 10); + msg.sessionId() = detail::createClientId(); + URI uri{ req.uri }; + msg.device() = uri.path()->substr(1, uri.path()->find('/', 1) - 1); + msg.property() = uri.path()->substr(uri.path()->find('/', 1) + 1); + msg.updateType() = static_cast(detail::UpdateType::NORMAL); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, detail::serialiseCmwLight(msg)); // send message header + if (auto params = uri.queryParamMap(); params.contains("ctx")) { + CmwLightRequestContext ctx; + ctx.selector() = params["ctx"].value_or("FAIR.SELECTOR.ALL"); + for (auto &[key, value] : params) { + if (key == "ctx") continue; + ctx.filters().insert({key, value.value_or("")}); + } + IoBuffer buffer{}; + serialise(buffer, ctx); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send context frame"sv, std::move(buffer)); // send requestContext + detail::send(con._socket, ZMQ_SNDMORE, "failed to send data frame"sv, std::move(req.data)); // send requestContext + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY_REQUEST_CONTEXT, BODY)); + } else { + detail::send(con._socket, ZMQ_SNDMORE, "failed to send data frame"sv, std::move(req.data)); // send requestContext + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY)); + } + req.state = RequestState::WAITING; + } + } + } + for (auto &sub : con._subscriptions | std::views::values) { + if (sub.state == INITIALIZED) { + detail::send(con._socket, ZMQ_SNDMORE, "error sending client req frame"sv, static_cast(detail::MessageType::CLIENT_REQ)); + URI uri{ sub.uri }; + CmwLightHeader header; + header.id() = sub.reqId; + header.device() = uri.path()->substr(1, uri.path()->find('/', 1) - 1); + header.property() = uri.path()->substr(uri.path()->find('/', 1) + 1); + header.requestType() = static_cast(detail::RequestType::SUBSCRIBE); + header.options() = std::make_unique(); + header.sessionId() = detail::createClientId(); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send message header"sv, detail::serialiseCmwLight(header)); // send message header + auto queryParams = uri.queryParamMap(); + CmwLightRequestContext ctx; + for (auto & [key, value] : queryParams) { + if (key == "ctx") { + ctx.selector() = queryParams.contains("ctx") ? queryParams["ctx"].value_or("") : ""; + } else { + ctx.filters().insert({key, value.value_or("")}); + } + } + IoBuffer buffer{}; + serialise(buffer, ctx); + detail::send(con._socket, ZMQ_SNDMORE, "failed to send context frame"sv, std::move(buffer)); // send requestContext + detail::send(con._socket, 0, "failed to send descriptor frame"sv, descriptorToString(HEADER, BODY_REQUEST_CONTEXT)); + con._lastHeartBeatSent = now; + sub.state = SUBSCRIBING; + } else if (sub.state == UNSUBSCRIBING) { + // TODO resend unsubscribe request on timeout, forcefully remove subscription after retries + } + } + if (con._lastHeartBeatSent < now - HEARTBEAT_INTERVAL) { + detail::send(con._socket, 0, "error sending connect frame"sv, static_cast(detail::MessageType::CLIENT_HB)); + con._lastHeartBeatSent = now; + } + if (con._lastHeartbeatReceived < now - HEARTBEAT_INTERVAL * 3) { + std::println("Missed 3 heartbeats -> connection seems to be broken"); // todo correct error handling + } + break; // do nothing + } + } + return now + _clientTimeout / 2; + } +}; + +/* + * Implementation of the Majordomo client protocol. Spawns a single thread which controls all client connections and sockets. + * A dispatcher thread reads the requests from the command ring buffer and dispatches them to the zeromq poll loop using an inproc socket pair. + * TODO: Q: merge with the mdp client? it basically uses the same loop and zeromq polling scheme. + */ +class CmwLightClientCtx : public ClientBase { + using timeUnit = std::chrono::milliseconds; + std::unordered_map, std::unique_ptr> _clients; + const zmq::Context &_zctx; + DirectoryLightClient _nameserver; + zmq::Socket _control_socket_send; + zmq::Socket _control_socket_recv; + std::jthread _poller; + std::vector _pollitems{}; + std::unordered_map _requests; + std::unordered_map _subscriptions; + timeUnit _timeout; + std::string _clientId; + std::size_t _request_id = 1; + +public: + explicit CmwLightClientCtx(const zmq::Context &zeromq_context, std::string nameserver, const timeUnit timeout = 100ms, std::string clientId = "") // todo: also pass thread pool + : _zctx{ zeromq_context }, _nameserver{DirectoryLightClient{std::move(nameserver)}}, _control_socket_send(zeromq_context, ZMQ_PAIR), _control_socket_recv(zeromq_context, ZMQ_PAIR), _timeout(timeout), _clientId(std::move(clientId)) { + zmq::invoke(zmq_bind, _control_socket_send, "inproc://mdclientControlSocket").assertSuccess(); + _pollitems.push_back({ .socket = _control_socket_recv.zmq_ptr, .fd = 0, .events = ZMQ_POLLIN, .revents = 0 }); + _poller = std::jthread([this](const std::stop_token &stoken) { this->poll(stoken); }); + } + + std::vector protocols() override { + return { "rda3", "rda3tcp" }; // rda3 protocol, if transport is unspecified, tcp is used if authority contains a port + } + + std::unique_ptr &getClient(const URI<> &uri) { + auto baseUri = URI<>::factory(uri).setQuery({}).path("").fragment("").build(); + if (_clients.contains(baseUri)) { + return _clients.at(baseUri); + } + auto [it, ins] = _clients.emplace(baseUri, createClient(baseUri, _nameserver)); + if (!ins) { + throw std::logic_error("could not insert client into client list"); + } + return it->second; + } + + std::unique_ptr createClient(const URI<> &uri, DirectoryLightClient &nameserver) { // todo: cleanup, uri is not used + return std::make_unique(_zctx, _pollitems, nameserver, _timeout, _clientId); + } + + void stop() override { + _poller.request_stop(); + _poller.join(); + } + + void request(Command cmd) override { + std::size_t req_id = 0; + if (cmd.command == mdp::Command::Get || cmd.command == mdp::Command::Set) { + req_id = _request_id++; + _requests.insert({ req_id, Request{ .uri = cmd.topic, .callback = std::move(cmd.callback), .timestamp_received = cmd.arrivalTime } }); + } else if (cmd.command == mdp::Command::Subscribe) { + req_id = _request_id++; + std::string clientRequestId = std::format("{}", req_id); + _subscriptions.insert({clientRequestId, Subscription{ .uri = cmd.topic, .callback = std::move(cmd.callback), .timestamp_received = cmd.arrivalTime, .reqId = req_id } }); + } else if (cmd.command == mdp::Command::Unsubscribe) { + std::string clientRequestId{}; + for (auto &[reqId, sub]: _subscriptions) { + if (sub.uri == cmd.topic) { + clientRequestId = reqId; + break; + } + } + if (const auto subIt = _subscriptions.find(clientRequestId); subIt != _subscriptions.end()) { + req_id = subIt->second.reqId;; + _subscriptions.erase(subIt); + }; + } + sendCmd(cmd.topic, cmd.command, req_id, cmd.data); + } + + DirectoryLightClient& nameserverClient() { + return _nameserver; + }; + +private: + void sendCmd(const URI<> &uri, mdp::Command commandType, std::size_t req_id, IoBuffer data = {}) const { + const bool isSet = commandType == mdp::Command::Set; + zmq::MessageFrame cmdType{ std::string{ static_cast(commandType) } }; + cmdType.send(_control_socket_send, ZMQ_SNDMORE).assertSuccess(); + zmq::MessageFrame reqId{ std::to_string(req_id) }; + reqId.send(_control_socket_send, ZMQ_SNDMORE).assertSuccess(); + zmq::MessageFrame endpoint{ std::string(uri.str()) }; + endpoint.send(_control_socket_send, isSet ? ZMQ_SNDMORE : 0).assertSuccess(); + if (isSet) { + zmq::MessageFrame dataframe{ std::move(data) }; + dataframe.send(_control_socket_send, 0).assertSuccess(); + } + } + + void handleRequests() { + zmq::MessageFrame cmd; + zmq::MessageFrame reqId; + zmq::MessageFrame endpoint; + while (cmd.receive(_control_socket_recv, ZMQ_DONTWAIT).isValid()) { + if (!reqId.receive(_control_socket_recv, ZMQ_DONTWAIT).isValid()) { + throw std::logic_error("invalid request received: failure receiving message"); + } + if (!endpoint.receive(_control_socket_recv, ZMQ_DONTWAIT).isValid()) { + throw std::logic_error("invalid request received: invalid message contents"); + } + URI uri{ std::string(endpoint.data()) }; + const auto &client = getClient(uri); + if (cmd.data().size() != 1) { + throw std::logic_error("invalid request received: wrong number of frames"); + } else if (cmd.data()[0] == static_cast(mdp::Command::Get)) { + client->get(uri, reqId.data()); + } else if (cmd.data()[0] == static_cast(mdp::Command::Set)) { + zmq::MessageFrame data; + if (!data.receive(_control_socket_recv, ZMQ_DONTWAIT).isValid()) { + throw std::logic_error("missing set data"); + } + client->set(uri, reqId.data(), std::span(data.data().data(), data.data().size())); + } else if (cmd.data()[0] == static_cast(mdp::Command::Subscribe)) { + client->subscribe(uri, reqId.data()); + } else if (cmd.data()[0] == static_cast(mdp::Command::Unsubscribe)) { + client->unsubscribe(uri, reqId.data()); + } else { + throw std::logic_error("invalid request received"); // messages always consist of 2 frames + } + } + } + + void poll(const std::stop_token &stoken) { + auto nextHousekeeping = std::chrono::system_clock::now(); + zmq::invoke(zmq_connect, _control_socket_recv, "inproc://mdclientControlSocket").assertSuccess(); + while (!stoken.stop_requested() && zmq::invoke(zmq_poll, _pollitems.data(), static_cast(_pollitems.size()), 200)) { + if (auto now = std::chrono::system_clock::now(); nextHousekeeping < now) { + nextHousekeeping = housekeeping(now); + // expire old subscriptions/requests/connections + } + handleRequests(); + for (const auto &client: _clients | std::views::values) { + mdp::Message receivedEvent; + while (client->receive(receivedEvent)) { + if (auto now = std::chrono::system_clock::now(); nextHousekeeping < now) { // perform housekeeping duties periodically + nextHousekeeping = housekeeping(now); + } + if (receivedEvent.command == mdp::Command::Invalid) { // Protocol internal messages, which require further receives but no publishing + continue; + } + std::string clientRequestId{reinterpret_cast(receivedEvent.clientRequestID.data()), receivedEvent.clientRequestID.size()}; + if (receivedEvent.command == mdp::Command::Notify && _subscriptions.contains(clientRequestId)) { + _subscriptions.at(clientRequestId).callback(receivedEvent); // callback + } + if (receivedEvent.command == mdp::Command::Final && _requests.contains(receivedEvent.id)) { + _requests.at(receivedEvent.id).callback(receivedEvent); // callback + _requests.erase(receivedEvent.id); + } + } + } + } + } + + timePoint housekeeping(const timePoint now) const { + timePoint next = now + _timeout; + for (const auto &client: _clients | std::views::values) { + next = std::min(next, client->housekeeping(now)); + } + return next; + } +}; +} // namespace opencmw::client::cmwlight +#endif // OPENCMW_CPP_CMWLIGHTCLIENT_HPP diff --git a/src/client/include/DirectoryLightClient.hpp b/src/client/include/DirectoryLightClient.hpp new file mode 100644 index 00000000..5e6020dc --- /dev/null +++ b/src/client/include/DirectoryLightClient.hpp @@ -0,0 +1,190 @@ +#ifndef OPENCMW_CPP_DIRECTORYLIGHTCLIENT_HPP +#define OPENCMW_CPP_DIRECTORYLIGHTCLIENT_HPP + +#include +#include +#include +#include + +#include + +#include + +#include + +namespace opencmw::client::cmwlight { + struct NameserverReplyLocation { + std::string domain; + std::string endpoint; + }; + struct NameserverReplyServer { + std::string name; + NameserverReplyLocation location; + }; + + struct NameserverReplyResource { + std::string name; + NameserverReplyServer server; + }; + + struct NameserverReply { + std::vector resources; + }; +} +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::NameserverReplyLocation, domain, endpoint) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::NameserverReplyServer, name, location) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::NameserverReplyResource, name, server) +ENABLE_REFLECTION_FOR(opencmw::client::cmwlight::NameserverReply, resources) + +/* + * Implements the CMW Directory Server lookup + * + * TODO: + * - allow more fields than just the address to be returned + * - how to communicate nameserver errors to the client + * - timeouts for queries and TTL for cache + * +* minimal API usage example +* curl ${CMW_NAMESERVER}/api/v1/devices/search --json '{ "proxyPreferred" : true, "domains" : [ ], "directServers" : [ ], "redirects" : { }, "names" : [ "GSITemplateDevice" ]}' +* {"resources":[{"name":"GSITemplateDevice","server":{"name":"GSITemplate_DU.vmla017","location":{"domain":"RDA3","endpoint":"9#Address:#string#19#tcp:%2F%2Fvmla017:36725#ApplicationId:#string#124#app=GSITemplate_DU;uid=root;host=vmla017;pid=398;os=Linux%2D6%2E6%2E111%2Drt31%2Dyocto%2Dpreempt%2Drt;osArch=64bit;appArch=64bit;lang=C%2B%2B;#Language:#string#3#C%2B%2B#Name:#string#22#GSITemplate_DU%2Evmla017#Pid:#int#398#ProcessName:#string#14#GSITemplate_DU#StartTime:#long#1771235547903#UserName:#string#4#root#Version:#string#5#5%2E1%2E1"}}}]} +*/ + +namespace opencmw::client::cmwlight { +class DirectoryLightClient { + std::map> cache; + std::mutex mutex; + std::deque pendingLookups; + std::string nameserver; + +public: + explicit DirectoryLightClient(std::string _nameserver) : nameserver(std::move(_nameserver)) {} + + std::optional lookup(const std::string_view name, const bool useCache = true) { + { + std::lock_guard lock{ mutex }; + if (useCache) { + if (const auto &res = cache.find(name); res != cache.end()) { + if (!res->second.empty()) { + return res->second; + } else { + return {}; // request was already sent + } + } + } + cache[std::string{name}] = ""; + } + triggerRequest(name); + return {}; + } + + void addStaticLookup(const std::string & deviceName, const std::string & address) { + std::lock_guard lock{ mutex }; + cache[deviceName] = address; + }; + + static std::optional parseEndpoint(std::string_view endpoint, const std::string &classname) { + using namespace std::literals; + auto urlDecode = [](const std::string &str) { + std::string ret; + ret.reserve(str.length()); + const std::size_t len = str.length(); + for (std::size_t i = 0; i < len; i++) { + if (str[i] != '%') { + if (str[i] == '+') { + ret += ' '; + } else { + ret += str[i]; + } + } else if (i + 2 < len) { + auto toHex = [](char c) { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + throw std::runtime_error("Invalid hexadecimal number"); + }; + const char ch = static_cast('\x10' * toHex(str.at(i + 1)) + toHex(str.at(i + 2))); + ret += ch; + i = i + 2; + } + } + return ret; + }; + auto tokens = std::views::split(endpoint, "#"sv); + if (tokens.empty()) { + return {}; + } + std::string fieldCountString = { tokens.front().data(), tokens.front().size() }; + char *end = to_address(fieldCountString.end()); + std::size_t fieldCount = std::strtoull(fieldCountString.data(), &end, 10); + + auto range = std::views::drop(tokens, 1); + auto iterator = range.begin(); + std::size_t n = 0; + while (n < fieldCount) { + std::string_view fieldNameView{ &(*iterator).front(), (*iterator).size() }; + std::string fieldname{ fieldNameView.substr(0, fieldNameView.size() - 1) }; + ++iterator; + if (std::string type{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; type == "string") { + ++iterator; + std::string sizeString{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; + auto parsed = std::to_address(sizeString.end()); + std::size_t size = std::strtoull(sizeString.data(), &parsed, 10); + ++iterator; + std::string string{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; + auto value = urlDecode(string); + if (fieldname == "Address") { + return value;; + } + } else if (type == "int") { + ++iterator; + std::string sizeString{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; + int number = std::atoi(sizeString.data()); + } else if (type == "long") { + ++iterator; + std::string sizeString{ std::string_view{ &(*iterator).front(), (*iterator).size() } }; + auto parsed = std::to_address(sizeString.end()); + long number = std::strtol(sizeString.data(), &parsed, 10); + } else { + FAIL(std::format("unknown type: {}, field: {}, endpoint: {}", type, fieldname, endpoint)); + } + ++iterator; + ++n; + } + return {}; + } + + static std::optional> parseNameserverReply(const std::string &reply) { + IoBuffer buffer{reply.data(), reply.size()}; + NameserverReply replyObj; + auto res = opencmw::deserialise(buffer, replyObj); + for (const auto &[name, server] : replyObj.resources) { + if (server.location.domain != "RDA3") { + continue; + } + if (auto address = parseEndpoint(server.location.endpoint, name)) { + return std::optional(std::make_pair(name, address.value())); + } + } + return {}; + } + + void triggerRequest(std::string_view name) { + using namespace std::literals; + std::string requestData = std::format(R"""({{ "proxyPreferred" : true, "domains" : [ ], "directServers" : [ ], "redirects" : {{ }}, "names" : [ "{}" ]}})""", name); + const auto uri = URI<>::factory(URI(std::string(nameserver))).path("/api/v1/devices/search").build(); + std::string deviceName{name}; + cpr::PostCallback([this, deviceName](cpr::Response response) -> void { + if (response.status_code == 200) { + if (std::optional> result = parseNameserverReply(response.text); result) { + if (deviceName != result->first) { + throw std::runtime_error("unexpected device name in reply"); + } + std::lock_guard lock{ mutex }; + cache[std::string{result->first}] = result->second; + } + } + }, cpr::Url{uri.str()}, cpr::Body{requestData}, cpr::Header{{"Content-Type", "application/json"}}); + } +}; +} +#endif // OPENCMW_CPP_DIRECTORYLIGHTCLIENT_HPP diff --git a/src/client/test/CMakeLists.txt b/src/client/test/CMakeLists.txt index ea4c7727..c4ef0cff 100644 --- a/src/client/test/CMakeLists.txt +++ b/src/client/test/CMakeLists.txt @@ -65,6 +65,16 @@ if(NOT EMSCRIPTEN) # TEST_PREFIX to whatever you want, or use different for different binaries catch_discover_tests(client_tests # TEST_PREFIX "unittests." REPORTER xml OUTPUT_DIR . OUTPUT_PREFIX "unittests." OUTPUT_SUFFIX .xml) catch_discover_tests(clientPublisher_tests) + + add_executable(CmwLightTest catch_main.cpp CmwLightTest.cpp) + target_link_libraries( + CmwLightTest + PUBLIC opencmw_project_warnings + opencmw_project_options + Catch2::Catch2 + client) + target_include_directories(CmwLightTest PRIVATE ${CMAKE_SOURCE_DIR}) + catch_discover_tests(CmwLightTest) endif() add_executable(rest_client_only_tests RestClientOnly_tests.cpp) diff --git a/src/client/test/CmwLightTest.cpp b/src/client/test/CmwLightTest.cpp new file mode 100644 index 00000000..fb2ce598 --- /dev/null +++ b/src/client/test/CmwLightTest.cpp @@ -0,0 +1,282 @@ +#include "MockServer.hpp" +#include "RestClientNative.hpp" +#include +#include +#include +#include +#include + +/** + * Most tests in this suite require a running Test Fesa Server which is registered to a Nameserver. + * Additionally, the following Environment Variables have to be set: + * - CMW_NAMESERVER=http://host:port - URL to the Nameserver to query + * - CMW_DEVICE_ADDRESS=tcp://host:port - URL where an instance of the GSITemplateDevice FESA class is running + * This could be done via nameserver lookup, but has to be set if the device needs ssh tunnelling with ssh -L local_port:device:port + * + * FESA Test Device: + * - vmla017 + * - DEV nameserver + * - Device: GSITemplateDevice + * - servername: GSITemplate_DU.vmla017 + * - Properties: + * - Acquisition: + * - Version + * - ModuleStatus + * - Status + * - Acquisition (mux) + * - Voltage: generiert jede 2 Sekunden Zufallswerte fuer Beam Process 1 und 2 + * - Setting: + * - Power + * - Init + * - Reset + * - Setting (mux) + * - Voltage: bestimmt die Acquisition-werte + * + * TODO: + * - test SET requests + */ +TEST_CASE("CmwNameserver", "[Client]") { + using namespace std::literals; + std::string nameserverExample = R"""(rda3://9#Address:#string#18#tcp:%2F%2Fdal025:16134#ApplicationId:#string#114#app=DigitizerDU2;uid=root;host=dal025;pid=16912;os=Linux%2D3%2E10%2E101%2Drt111%2Dscu03;osArch=64bit;appArch=64bit;lang=C%2B%2B;#Language:#string#3#C%2B%2B#Name:#string#19#DigitizerDU2%2Edal025#Pid:#int#16912#ProcessName:#string#12#DigitizerDU2#StartTime:#long#1699343695922#UserName:#string#4#root#Version:#string#5#3%2E1%2E0)"""; + std::string nameserverExample2 = R"""(rda3://9#Address:#string#18#tcp:%2F%2Ffel0053:3717#ApplicationId:#string#115#app=DigitizerDU2;uid=root;host=fel0053;pid=31447;os=Linux%2D3%2E10%2E101%2Drt111%2Dscu03;osArch=64bit;appArch=64bit;lang=C%2B%2B;#Language:#string#3#C%2B%2B#Name:#string#20#DigitizerDU2%2Efel0053#Pid:#int#31447#ProcessName:#string#12#DigitizerDU2#StartTime:#long#1701529074225#UserName:#string#4#root#Version:#string#5#3%2E1%2E0)"""; + std::string nameserverReply = R"""({"resources":[{"name":"GECD001","server":{"name":"DigitizerDU2.dal018","location":{"domain":"RDA3","endpoint":"9#Address:#string#17#tcp:%2F%2Fdal018:6397#ApplicationId:#string#113#app=DigitizerDU2;uid=root;host=dal018;pid=1977;os=Linux%2D3%2E10%2E101%2Drt111%2Dscu03;osArch=64bit;appArch=64bit;lang=C%2B%2B;#Language:#string#3#C%2B%2B#Name:#string#19#DigitizerDU2%2Edal018#Pid:#int#1977#ProcessName:#string#12#DigitizerDU2#StartTime:#long#1769161872191#UserName:#string#4#root#Version:#string#5#3%2E1%2E0"}}}]})"""; + + SECTION("ParseNameserverReply") { + auto result = opencmw::client::cmwlight::DirectoryLightClient::parseNameserverReply(nameserverReply); + REQUIRE(result.has_value()); + REQUIRE(result->second == R"""(tcp://dal018:6397)"""); + } + + SECTION("Query rda3 directory server/nameserver") { + auto env_nameserver = std::getenv("CMW_NAMESERVER"); + if (env_nameserver == nullptr) { + std::println("skipping BasicCmwLight example test as it relies on the availability of network infrastructure."); + return; // skip test + } + std::string nameserver{ env_nameserver }; + const opencmw::zmq::Context zctx{}; + opencmw::client::cmwlight::DirectoryLightClient nameserverClient{nameserver}; + std::optional result = nameserverClient.lookup("GSITemplateDevice"); + REQUIRE(!result.has_value()); // check that the device is originally not in the nameserver's cache + while (!(result = nameserverClient.lookup("GSITemplateDevice")).has_value()) { + std::this_thread::sleep_for(1ms); + } + REQUIRE(result.value().starts_with("tcp://vmla")); + // check that later lookups immediately return the cached result + result = nameserverClient.lookup("GSITemplateDevice"); + REQUIRE(result.value().starts_with("tcp://vmla")); + }; +} + +// small utility function that prints the content of a string in the classic hexedit way with address, hexadecimal and ascii representations +static std::string hexview(const std::string_view value, std::size_t bytesPerLine = 4) { + std::string result; + result.reserve(value.size() * 4); + std::string alpha; // temporarily store the ascii representation + alpha.reserve(8 * bytesPerLine); + std::size_t i = 0; + for (auto c : value) { + if (i % (bytesPerLine * 8) == 0) { + result.append(std::format("{0:#08x} - {0:04} | ", i)); // print address in hex and decimal + } + result.append(std::format("{:02x} ", c)); + alpha.append(std::format("{}", std::isprint(c) ? c : '.')); + if ((i + 1) % 8 == 0) { + result.append(" "); + alpha.append(" "); + } + if ((i + 1) % (bytesPerLine * 8) == 0) { + result.append(std::format(" {}\n", alpha)); + alpha.clear(); + } + i++; + } + result.append(std::format("{:{}} {}\n", "", 3 * (9 * bytesPerLine - alpha.size()), alpha)); + return result; +} + +static bool waitFor(const std::atomic &counter, const int expectedValue, const auto timeout) { + const auto start = std::chrono::system_clock::now(); + while (counter.load() < expectedValue && std::chrono::system_clock::now() - start < timeout) { + std::this_thread::sleep_for(std::chrono::milliseconds(100)); + } + return counter.load() == expectedValue; +} +static bool waitFor(const std::atomic &counter, const int expectedValue) { + using namespace std::literals; + return waitFor(counter, expectedValue, 1000s); +} + +TEST_CASE("CmwLightClientGet", "[Client]") { + using namespace opencmw; + using namespace std::literals; + const std::string DEVICE_NAME = "GSITemplateDevice"; + const std::string STATUS_PROPERTY = "/GSITemplateDevice/Status"; + const std::string ACQUISITION_PROPERTY = "/GSITemplateDevice/Acquisition"; + const std::string SELECTOR = "FAIR.SELECTOR.C=3:S=1:P=1:T=300"; + + char * env_nameserver = std::getenv("CMW_NAMESERVER"); + if (env_nameserver == nullptr) { + std::println("skipping BasicCmwLight example test as it relies on the availability of network infrastructure. Define CMW_NAMESERVER environment variable to run this test."); + return; // skip test + } + std::string nameserver{ env_nameserver }; + const zmq::Context zctx{}; + std::vector> clients; + auto cmwlightClient = std::make_unique(zctx, nameserver, 100ms, "testclient"); + std::string deviceHost{}; // "tcp://vmla017:36725" }; + if (char * env_cmw_host = std::getenv("CMW_DEVICE_HOST"); env_cmw_host != nullptr) { + deviceHost = std::string{env_cmw_host}; + cmwlightClient->nameserverClient().addStaticLookup(DEVICE_NAME, std::string{env_cmw_host}); + } else { + while (!deviceHost.empty()) { + deviceHost = cmwlightClient->nameserverClient().lookup(DEVICE_NAME).value_or(""); + } + } + clients.emplace_back(std::move(cmwlightClient)); + client::ClientContext clientContext{ std::move(clients) }; + // send some requests + { + auto endpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(STATUS_PROPERTY).build(); + std::atomic getReceived{ 0 }; + clientContext.get(endpoint, [&getReceived](const mdp::Message &message) { + if (!message.error.empty()) { + FAIL("get should have succeeded"); + } else { + ++getReceived; + } + }); + REQUIRE(waitFor(getReceived, 1)); + } + { + auto endpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(ACQUISITION_PROPERTY).addQueryParameter("ctx", SELECTOR).build(); + std::atomic getReceived{ 0 }; + clientContext.get(endpoint, [&getReceived](const mdp::Message &message) { + if (!message.error.empty()) { + FAIL("get should have succeeded"); + } else { + ++getReceived; + } + }); + REQUIRE(waitFor(getReceived, 1)); + } + { + auto endpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path("/GSITemplateDevice/Setting").build(); + std::atomic getError{ 0 }; + clientContext.get(endpoint, [&getError](const mdp::Message &message) { + if (!message.error.empty()) { + REQUIRE(message.error.contains("Access point 'GSITemplateDevice/Setting' needs a selector")); + ++getError; + } else { + FAIL("get should have failed"); + } + }); + REQUIRE(waitFor(getError, 1)); + } + { // SET Request + + } + { // Unreachable device server + + } + { // non-existent device property + + } +} + +TEST_CASE("CmwLightClientSubscribe", "[Client]") { + using namespace opencmw; + using namespace std::literals; + const std::string DEVICE_NAME = "GSITemplateDevice"; + const std::string STATUS_PROPERTY = "/GSITemplateDevice/Status"; + const std::string ACQUISITION_PROPERTY = "/GSITemplateDevice/Acquisition"; + const std::string SELECTOR = "FAIR.SELECTOR.P=1"; + + char * env_nameserver = std::getenv("CMW_NAMESERVER"); + if (env_nameserver == nullptr) { + std::println("skipping BasicCmwLight example test as it relies on the availability of network infrastructure. Define CMW_NAMESERVER environment variable to run this test."); + return; // skip test + } + std::string nameserver{ env_nameserver }; + const zmq::Context zctx{}; + std::vector> clients; + auto cmwlightClient = std::make_unique(zctx, nameserver, 100ms, "testclient"); + std::string deviceHost{}; // "tcp://vmla017:36725" }; + if (char * env_cmw_host = std::getenv("CMW_DEVICE_HOST"); env_cmw_host != nullptr) { + deviceHost = std::string{env_cmw_host}; + cmwlightClient->nameserverClient().addStaticLookup(DEVICE_NAME, std::string{env_cmw_host}); + } else { + while (!deviceHost.empty()) { + deviceHost = cmwlightClient->nameserverClient().lookup(DEVICE_NAME).value_or(""); + } + } + clients.emplace_back(std::move(cmwlightClient)); + client::ClientContext clientContext{ std::move(clients) }; + // setup subscription + { // Subscribe non-multiplexed + std::atomic subscriptionUpdatesReceived{ 0 }; + auto subscriptionEndpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(STATUS_PROPERTY).build(); + clientContext.subscribe(subscriptionEndpoint, [&subscriptionUpdatesReceived](const mdp::Message &message) { + if (!message.error.empty()) { + FAIL("subscription should not notify exceptions"); + } else { + IoBuffer buffer(message.data); + majordomo::Empty empty{}; + auto deserialiserInfo = deserialise(buffer, empty); // deserialising into empty struct to get field information + //std::println("Deserialised subscription reply:\n fields: {}\n fieldTypes: {}", deserialiserInfo.additionalFields | std::views::keys, deserialiserInfo.additionalFields | std::views::values); + REQUIRE(deserialiserInfo.additionalFields.size() == 13); + ++subscriptionUpdatesReceived; + } + }); + REQUIRE(waitFor(subscriptionUpdatesReceived, 2, 5s)); // property gets notified every 2 seconds + // Check that subscription updates stop after unsubscribing + clientContext.unsubscribe(subscriptionEndpoint); + std::this_thread::sleep_for(200ms); // get a few subscription updates + int subscriptionUpdatesAfterUnsubscribe = subscriptionUpdatesReceived; + std::this_thread::sleep_for(5s); // get a few subscription updates + REQUIRE(subscriptionUpdatesReceived == subscriptionUpdatesAfterUnsubscribe); + } + { // error case: subscribe to multiplexed without selector + std::atomic subscriptionUpdateError{ 0 }; + auto subscriptionEndpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(ACQUISITION_PROPERTY).build(); + clientContext.subscribe(subscriptionEndpoint, [&subscriptionUpdateError](const mdp::Message &message) { + if (!message.error.empty()) { + ++subscriptionUpdateError; + REQUIRE(message.error.contains("Access point 'GSITemplateDevice/Acquisition' needs a selector")); + } else { + FAIL("invalid subscription should not notify updates"); + } + }); + REQUIRE(waitFor(subscriptionUpdateError, 1, 5s)); // property gets notified every 2 seconds + // TODO: does this need unsubscribe or is the error enough? does it do auto retry? + } + { // subscribe to muliplexed property + std::atomic subscriptionUpdatesReceived{ 0 }; + auto subscriptionEndpoint = URI<>::factory(URI(deviceHost)).scheme("rda3tcp").path(ACQUISITION_PROPERTY).addQueryParameter("ctx", SELECTOR).build(); + clientContext.subscribe(subscriptionEndpoint, [&subscriptionUpdatesReceived](const mdp::Message &message) { + if (!message.error.empty()) { + FAIL("subscription should not notify exceptions"); + } else { + IoBuffer buffer(message.data); + majordomo::Empty empty{}; + auto deserialiserInfo = deserialise(buffer, empty); // deserialising into empty struct to get field information + //std::println("Deserialised subscription reply:\n fields: {}\n fieldTypes: {}", deserialiserInfo.additionalFields | std::views::keys, deserialiserInfo.additionalFields | std::views::values); + REQUIRE(deserialiserInfo.additionalFields.size() == 12); + ++subscriptionUpdatesReceived; + } + }); + REQUIRE(waitFor(subscriptionUpdatesReceived, 2, 5s)); // property gets notified every 2 seconds + // Check that subscription updates stop after unsubscribing + clientContext.unsubscribe(subscriptionEndpoint); + std::this_thread::sleep_for(200ms); // get a few subscription updates + int subscriptionUpdatesAfterUnsubscribe = subscriptionUpdatesReceived; + std::this_thread::sleep_for(5s); // get a few subscription updates + REQUIRE(subscriptionUpdatesReceived == subscriptionUpdatesAfterUnsubscribe); + } + // SECTION("ErrorCase unreachable device server") {} + // SECTION("ErrorCase missing selector for multiplexed Property") {} + + // test filters? + // filters2String = "acquisitionModeFilter=int:0&channelNameFilter=GS11MU2:Voltage_1@10Hz"; + // subscribe("r1", new URI("rda3", null, '/' + DEVICE + '/' + PROPERTY, "ctx=" + SELECTOR + "&" + filtersString, null), null); +} diff --git a/src/core/include/Topic.hpp b/src/core/include/Topic.hpp index 6a851cb0..3fc228fa 100644 --- a/src/core/include/Topic.hpp +++ b/src/core/include/Topic.hpp @@ -103,11 +103,13 @@ struct Topic { return opencmw::URI::factory().path(_service).setQuery({ _params.begin(), _params.end() }).build(); } - std::string toZmqTopic() const { + std::string toZmqTopic(const bool delimitWithPound = true) const { using namespace std::string_literals; std::string zmqTopic = _service; if (_params.empty()) { - zmqTopic += "#"s; + if (delimitWithPound) { + zmqTopic += "#"s; + } return zmqTopic; } zmqTopic += "?"s; @@ -123,7 +125,9 @@ struct Topic { } isFirst = false; } - zmqTopic += "#"s; + if (delimitWithPound) { + zmqTopic += "#"s; + } return zmqTopic; } diff --git a/src/zmq/include/zmq/ZmqUtils.hpp b/src/zmq/include/zmq/ZmqUtils.hpp index 209bb369..d50571ba 100644 --- a/src/zmq/include/zmq/ZmqUtils.hpp +++ b/src/zmq/include/zmq/ZmqUtils.hpp @@ -231,6 +231,16 @@ class MessageFrame { copy); } + explicit MessageFrame(char c) { + const auto copy = new char[1] {c}; + zmq_msg_init_data( + &_message, copy, 1, + [](void * /*unused*/, void *bufOwned) { + delete static_cast(bufOwned); + }, + copy); + } + static MessageFrame fromStaticData(std::string_view buf) { MessageFrame mf; zmq_msg_init_data(