From cd1108822fc1a95b39dc8825e99a0b7b515ae585 Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Fri, 6 Mar 2026 15:40:31 +0100 Subject: [PATCH 01/15] warning fixes --- modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp index cc3dd3d6..7b5ff7ce 100644 --- a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp @@ -307,7 +307,7 @@ void MqttPublisherFbImpl::updateCoreEventCallbacks() } else { - signalMap.insert(std::move(signalMapCopy.extract(sig.getGlobalId().toStdString()))); + signalMap.insert(signalMapCopy.extract(sig.getGlobalId().toStdString())); } } ++it; @@ -327,7 +327,7 @@ void MqttPublisherFbImpl::clearCoreEventCallbacks(const std::unordered_map& errors) + auto buildErrorString = [](const std::vector& errors) { std::string allMessages; for (const auto& msg : errors) From 4208ed979b814b60c5ad0d39d88d52fbe7337736 Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Fri, 6 Mar 2026 16:53:56 +0100 Subject: [PATCH 02/15] new API for MqttDataSample --- .../atomic_signal_atomic_sample_handler.h | 2 +- .../atomic_signal_sample_arr_handler.h | 2 +- .../include/mqtt_streaming_module/types.h | 53 +++++++++++++++++-- .../atomic_signal_atomic_sample_handler.cpp | 4 +- .../src/atomic_signal_sample_arr_handler.cpp | 6 +-- .../group_signal_shared_ts_arr_handler.cpp | 3 +- .../src/group_signal_shared_ts_handler.cpp | 3 +- .../src/mqtt_publisher_fb_impl.cpp | 13 ++--- .../src/signal_arr_atomic_sample_handler.cpp | 3 +- 9 files changed, 70 insertions(+), 19 deletions(-) diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_atomic_sample_handler.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_atomic_sample_handler.h index 22090bd2..4615ea1b 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_atomic_sample_handler.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_atomic_sample_handler.h @@ -39,7 +39,7 @@ class AtomicSignalAtomicSampleHandler : public HandlerBase virtual MqttData processSignalContext(SignalContext& signalContext); void processSignalDescriptorChanged(SignalContext& signalCtx, const DataDescriptorPtr& valueSigDesc, const DataDescriptorPtr& domainSigDesc); - MqttDataSample processDataPacket(SignalContext& signalContext, const DataPacketPtr& dataPacket, size_t offset); + MqttDataSamplePtr processDataPacket(SignalContext& signalContext, const DataPacketPtr& dataPacket, size_t offset); std::string toString(const std::string valueFieldName, daq::DataPacketPtr packet, size_t offset); std::string buildTopicName(const SignalContext& signalContext); diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_sample_arr_handler.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_sample_arr_handler.h index d0179573..c61d77e3 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_sample_arr_handler.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_sample_arr_handler.h @@ -47,7 +47,7 @@ class AtomicSignalSampleArrayHandler : public AtomicSignalAtomicSampleHandler std::unordered_map signalBuffers; MqttData processSignalContext(SignalContext& signalContext) override; - MqttDataSample processDataPackets(SignalContext& signalContext); + MqttDataSamplePtr processDataPackets(SignalContext& signalContext); std::string toString(const std::string valueFieldName, SignalContext& signalContext); std::pair getSample(SignalContext& signalContext); void diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/types.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/types.h index 26e62554..d416a50d 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/types.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/types.h @@ -7,14 +7,61 @@ BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE -struct MqttDataSample { + +class IMqttDataSample { +public: + IMqttDataSample(){}; + IMqttDataSample(SignalConfigPtr sigConfPtr, std::string&& topic) + : previewSignal(std::move(sigConfPtr)), + topic(std::move(topic)) + { + } + + virtual ~IMqttDataSample() = default; + SignalConfigPtr getPreviewSignal() const + { + return previewSignal; + } + std::string getTopic() const + { + return topic; + } + virtual void* getDataPointer() const = 0; + virtual size_t getDataSize() const = 0; + +protected: SignalConfigPtr previewSignal; std::string topic; - std::string message; +}; + +using MqttDataSamplePtr = std::shared_ptr; + +template +class MqttDataSample : public IMqttDataSample { +public: + MqttDataSample(){}; + MqttDataSample(SignalConfigPtr sigConfPtr, std::string&& topic, T&& message) + : IMqttDataSample(std::move(sigConfPtr), std::move(topic)), + message(std::move(message)) + { + } + + void* getDataPointer() const override + { + return const_cast(reinterpret_cast(message.data())); + } + + size_t getDataSize() const override + { + return message.size(); + } + +protected: + T message; }; struct MqttData { - std::vector data; + std::vector data; bool needRevalidation = false; void merge(MqttData&& other) diff --git a/modules/mqtt_streaming_module/src/atomic_signal_atomic_sample_handler.cpp b/modules/mqtt_streaming_module/src/atomic_signal_atomic_sample_handler.cpp index aba74528..a0922109 100644 --- a/modules/mqtt_streaming_module/src/atomic_signal_atomic_sample_handler.cpp +++ b/modules/mqtt_streaming_module/src/atomic_signal_atomic_sample_handler.cpp @@ -136,13 +136,13 @@ std::string AtomicSignalAtomicSampleHandler::buildTopicName(const SignalContext& return signalContext.inputPort.getSignal().getGlobalId().toStdString(); } -MqttDataSample AtomicSignalAtomicSampleHandler::processDataPacket(SignalContext& signalContext, const DataPacketPtr& dataPacket, size_t offset) +MqttDataSamplePtr AtomicSignalAtomicSampleHandler::processDataPacket(SignalContext& signalContext, const DataPacketPtr& dataPacket, size_t offset) { const auto signal = signalContext.inputPort.getSignal(); std::string valueFieldName = buildValueFieldName(signalNamesMode, signal); auto msg = toString(valueFieldName, dataPacket, offset); std::string topic = buildTopicName(signalContext); - return MqttDataSample{signalContext.previewSignal, topic, msg}; + return std::make_shared>(signalContext.previewSignal, std::move(topic), std::move(msg)); } ListPtr AtomicSignalAtomicSampleHandler::getTopics(const std::vector& signalContexts) diff --git a/modules/mqtt_streaming_module/src/atomic_signal_sample_arr_handler.cpp b/modules/mqtt_streaming_module/src/atomic_signal_sample_arr_handler.cpp index 0e8f32a6..0f24117f 100644 --- a/modules/mqtt_streaming_module/src/atomic_signal_sample_arr_handler.cpp +++ b/modules/mqtt_streaming_module/src/atomic_signal_sample_arr_handler.cpp @@ -154,14 +154,14 @@ std::string AtomicSignalSampleArrayHandler::toString(const std::string valueFiel return result; } -MqttDataSample AtomicSignalSampleArrayHandler::processDataPackets(SignalContext& signalContext) +MqttDataSamplePtr AtomicSignalSampleArrayHandler::processDataPackets(SignalContext& signalContext) { if (signalBuffers[signalContext.inputPort.getSignal().getGlobalId().toStdString()].data.empty()) - return MqttDataSample{nullptr, "", ""}; + std::make_shared>(); const auto signal = signalContext.inputPort.getSignal(); std::string valueFieldName = buildValueFieldName(signalNamesMode, signal); auto msg = toString(valueFieldName, signalContext); std::string topic = buildTopicName(signalContext); - return MqttDataSample{signalContext.previewSignal, topic, msg}; + return std::make_shared>(signalContext.previewSignal, std::move(topic), std::move(msg)); } END_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE diff --git a/modules/mqtt_streaming_module/src/group_signal_shared_ts_arr_handler.cpp b/modules/mqtt_streaming_module/src/group_signal_shared_ts_arr_handler.cpp index 99603813..40eaffda 100644 --- a/modules/mqtt_streaming_module/src/group_signal_shared_ts_arr_handler.cpp +++ b/modules/mqtt_streaming_module/src/group_signal_shared_ts_arr_handler.cpp @@ -51,7 +51,8 @@ MqttData GroupSignalSharedTsArrHandler::processSignalContexts(std::vector>(signalContexts[0].previewSignal, buildTopicName(), messageFromFields(fields))); } return messages; } diff --git a/modules/mqtt_streaming_module/src/group_signal_shared_ts_handler.cpp b/modules/mqtt_streaming_module/src/group_signal_shared_ts_handler.cpp index f83c10df..b3f794a4 100644 --- a/modules/mqtt_streaming_module/src/group_signal_shared_ts_handler.cpp +++ b/modules/mqtt_streaming_module/src/group_signal_shared_ts_handler.cpp @@ -52,7 +52,8 @@ MqttData GroupSignalSharedTsHandler::processSignalContexts(std::vector>(signalContexts[0].previewSignal, std::move(topic), std::move(msg))); } } diff --git a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp index 7b5ff7ce..920de0cb 100644 --- a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp @@ -558,15 +558,16 @@ void MqttPublisherFbImpl::readerLoop() void MqttPublisherFbImpl::sendMessages(const MqttData& data) { - for (const auto& [signal, topic, msg] : data.data) + for (const auto& samplePtr : data.data) { - if (signal.assigned()) + auto prevSig = samplePtr->getPreviewSignal(); + if (prevSig.assigned()) { - const auto outputPacket = BinaryDataPacket(nullptr, signal.getDescriptor(), msg.size()); - memcpy(outputPacket.getData(), msg.data(), msg.size()); - signal.sendPacket(outputPacket); + const auto outputPacket = BinaryDataPacket(nullptr, prevSig.getDescriptor(), samplePtr->getDataSize()); + memcpy(outputPacket.getData(), samplePtr->getDataPointer(), samplePtr->getDataSize()); + prevSig.sendPacket(outputPacket); } - auto status = mqttClient->publish(topic, (void*)msg.c_str(), msg.length(), config.qos); + auto status = mqttClient->publish(samplePtr->getTopic(), samplePtr->getDataPointer(), samplePtr->getDataSize(), config.qos); if (!status.success) { hasSkippedMsg = true; diff --git a/modules/mqtt_streaming_module/src/signal_arr_atomic_sample_handler.cpp b/modules/mqtt_streaming_module/src/signal_arr_atomic_sample_handler.cpp index edc79bde..6ee7d6cc 100644 --- a/modules/mqtt_streaming_module/src/signal_arr_atomic_sample_handler.cpp +++ b/modules/mqtt_streaming_module/src/signal_arr_atomic_sample_handler.cpp @@ -49,7 +49,8 @@ MqttData SignalArrayAtomicSampleHandler::processSignalContexts(std::vector>(signalContexts[0].previewSignal, std::move(topic), std::move(msg))); } return messages; } From 6cd9405d0a75a98cf507924e133eac9789098ac4 Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Fri, 6 Mar 2026 17:55:59 +0100 Subject: [PATCH 03/15] RawHandler class --- .../atomic_signal_atomic_sample_handler.h | 2 + .../group_signal_shared_ts_handler.h | 1 + .../mqtt_streaming_module/handler_base.h | 6 +- .../mqtt_streaming_module/raw_handler.h | 46 ++++ .../signal_arr_atomic_sample_handler.h | 1 + .../mqtt_streaming_module/src/CMakeLists.txt | 4 + .../atomic_signal_atomic_sample_handler.cpp | 3 +- .../src/group_signal_shared_ts_handler.cpp | 5 +- .../mqtt_streaming_module/src/raw_handler.cpp | 199 ++++++++++++++++++ .../src/signal_arr_atomic_sample_handler.cpp | 5 +- 10 files changed, 263 insertions(+), 9 deletions(-) create mode 100644 modules/mqtt_streaming_module/include/mqtt_streaming_module/raw_handler.h create mode 100644 modules/mqtt_streaming_module/src/raw_handler.cpp diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_atomic_sample_handler.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_atomic_sample_handler.h index 4615ea1b..f1005024 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_atomic_sample_handler.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/atomic_signal_atomic_sample_handler.h @@ -36,6 +36,8 @@ class AtomicSignalAtomicSampleHandler : public HandlerBase ListPtr getTopics(const std::vector& signalContexts) override; std::string getSchema() override; protected: + SignalValueJSONKey signalNamesMode; + virtual MqttData processSignalContext(SignalContext& signalContext); void processSignalDescriptorChanged(SignalContext& signalCtx, const DataDescriptorPtr& valueSigDesc, const DataDescriptorPtr& domainSigDesc); diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/group_signal_shared_ts_handler.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/group_signal_shared_ts_handler.h index 34bcc6b1..46bee784 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/group_signal_shared_ts_handler.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/group_signal_shared_ts_handler.h @@ -54,6 +54,7 @@ class GroupSignalSharedTsHandler : public HandlerBase protected: const size_t buffersSize; const std::string topic; + SignalValueJSONKey signalNamesMode; std::vector dataBuffers; bool firstDescriptorChange; daq::MultiReaderPtr reader; diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/handler_base.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/handler_base.h index c68fe5a6..0b6d9a4c 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/handler_base.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/handler_base.h @@ -27,9 +27,8 @@ BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE class HandlerBase { public: - HandlerBase(WeakRefPtr parentFb, SignalValueJSONKey signalNamesMode) - : parentFb(parentFb), - signalNamesMode(signalNamesMode) + HandlerBase(WeakRefPtr parentFb) + : parentFb(parentFb) { } virtual ~HandlerBase() = default; @@ -58,7 +57,6 @@ class HandlerBase protected: WeakRefPtr parentFb; - SignalValueJSONKey signalNamesMode; static std::pair calculateRatio(const DataDescriptorPtr descriptor) { diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/raw_handler.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/raw_handler.h new file mode 100644 index 00000000..71272edc --- /dev/null +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/raw_handler.h @@ -0,0 +1,46 @@ +/* + * Copyright 2022-2025 openDAQ d.o.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once + +#include +#include + +BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE + +class RawHandler : public HandlerBase +{ +public: + explicit RawHandler(WeakRefPtr parentFb); + + MqttData processSignalContexts(std::vector& signalContexts) override; + ProcedureStatus validateSignalContexts(const std::vector& signalContexts) const override; + ProcedureStatus signalListChanged(std::vector&) override + { + return ProcedureStatus{true, {}}; + }; + ListPtr getTopics(const std::vector& signalContexts) override; + std::string getSchema() override; +protected: + virtual MqttData processSignalContext(SignalContext& signalContext); + void + processSignalDescriptorChanged(SignalContext& signalCtx, const DataDescriptorPtr& valueSigDesc, const DataDescriptorPtr& domainSigDesc); + MqttDataSamplePtr processDataPacket(SignalContext& signalContext, const DataPacketPtr& dataPacket, size_t offset); + std::string buildTopicName(const SignalContext& signalContext); + std::vector toDataBuffer(daq::DataPacketPtr packet, size_t offset); +}; + +END_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/signal_arr_atomic_sample_handler.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/signal_arr_atomic_sample_handler.h index 545f9009..c62307f2 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/signal_arr_atomic_sample_handler.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/signal_arr_atomic_sample_handler.h @@ -36,6 +36,7 @@ class SignalArrayAtomicSampleHandler : public HandlerBase protected: const std::string topic; + SignalValueJSONKey signalNamesMode; std::string toString(const std::string valueFieldName, daq::DataPacketPtr packet); std::string buildTopicName(); diff --git a/modules/mqtt_streaming_module/src/CMakeLists.txt b/modules/mqtt_streaming_module/src/CMakeLists.txt index f3859381..88a48d73 100644 --- a/modules/mqtt_streaming_module/src/CMakeLists.txt +++ b/modules/mqtt_streaming_module/src/CMakeLists.txt @@ -17,6 +17,7 @@ set(SRC_Include common.h group_signal_shared_ts_handler.h group_signal_shared_ts_arr_handler.h signal_arr_atomic_sample_handler.h + raw_handler.h types.h status_helper.h status_adaptor.h @@ -34,6 +35,7 @@ set(SRC_Srcs module_dll.cpp group_signal_shared_ts_handler.cpp group_signal_shared_ts_arr_handler.cpp signal_arr_atomic_sample_handler.cpp + raw_handler.cpp ) opendaq_prepend_include(${TARGET_FOLDER_NAME} SRC_Include) @@ -70,11 +72,13 @@ source_group("handlers" FILES ${MODULE_HEADERS_DIR}/handler_base.h ${MODULE_HEADERS_DIR}/group_signal_shared_ts_handler.h ${MODULE_HEADERS_DIR}/group_signal_shared_ts_arr_handler.h ${MODULE_HEADERS_DIR}/signal_arr_atomic_sample_handler.h + ${MODULE_HEADERS_DIR}/raw_handler.h atomic_signal_atomic_sample_handler.cpp atomic_signal_sample_arr_handler.cpp group_signal_shared_ts_handler.cpp group_signal_shared_ts_arr_handler.cpp signal_arr_atomic_sample_handler.cpp + raw_handler.cpp ) find_package(Boost REQUIRED COMPONENTS algorithm) diff --git a/modules/mqtt_streaming_module/src/atomic_signal_atomic_sample_handler.cpp b/modules/mqtt_streaming_module/src/atomic_signal_atomic_sample_handler.cpp index a0922109..4a8721c5 100644 --- a/modules/mqtt_streaming_module/src/atomic_signal_atomic_sample_handler.cpp +++ b/modules/mqtt_streaming_module/src/atomic_signal_atomic_sample_handler.cpp @@ -8,7 +8,8 @@ BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE AtomicSignalAtomicSampleHandler::AtomicSignalAtomicSampleHandler(WeakRefPtr parentFb, SignalValueJSONKey signalNamesMode) - : HandlerBase(parentFb, signalNamesMode) + : HandlerBase(parentFb), + signalNamesMode(signalNamesMode) { } diff --git a/modules/mqtt_streaming_module/src/group_signal_shared_ts_handler.cpp b/modules/mqtt_streaming_module/src/group_signal_shared_ts_handler.cpp index b3f794a4..0c82aa8c 100644 --- a/modules/mqtt_streaming_module/src/group_signal_shared_ts_handler.cpp +++ b/modules/mqtt_streaming_module/src/group_signal_shared_ts_handler.cpp @@ -11,9 +11,10 @@ BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE GroupSignalSharedTsHandler::GroupSignalSharedTsHandler(WeakRefPtr parentFb, SignalValueJSONKey signalNamesMode, std::string topic) - : HandlerBase(parentFb, signalNamesMode), + : HandlerBase(parentFb), buffersSize(1000), - topic(topic) + topic(topic), + signalNamesMode(signalNamesMode) { } diff --git a/modules/mqtt_streaming_module/src/raw_handler.cpp b/modules/mqtt_streaming_module/src/raw_handler.cpp new file mode 100644 index 00000000..3d1e1d91 --- /dev/null +++ b/modules/mqtt_streaming_module/src/raw_handler.cpp @@ -0,0 +1,199 @@ +#include +#include +#include +#include +#include +#include +#include + +BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE + +RawHandler::RawHandler(WeakRefPtr parentFb) + : HandlerBase(parentFb) +{ +} + +MqttData RawHandler::processSignalContexts(std::vector& signalContexts) +{ + MqttData messages; + for (auto& sigCtx : signalContexts) + { + auto msgs = processSignalContext(sigCtx); + messages.merge(std::move(msgs)); + } + return messages; +} + +ProcedureStatus RawHandler::validateSignalContexts(const std::vector& signalContexts) const +{ + static const std::set allowedSampleTypes{SampleType::Float64, + SampleType::Float32, + SampleType::UInt8, + SampleType::Int8, + SampleType::UInt16, + SampleType::Int16, + SampleType::UInt32, + SampleType::Int32, + SampleType::UInt64, + SampleType::Int64, + SampleType::Binary, + SampleType::String}; + ProcedureStatus status{true, {}}; + for (const auto& sigCtx : signalContexts) + { + auto signal = sigCtx.inputPort.getSignal(); + if (!signal.assigned()) + continue; + if (!signal.getDescriptor().assigned()) + { + status.addError(fmt::format("Connected signal \"{}\" doesn't contain a descroptor. This is not allowed.", + sigCtx.inputPort.getSignal().getGlobalId())); + } + if (auto demensions = signal.getDescriptor().getDimensions(); demensions.assigned() && demensions.getCount() > 0) + { + status.addError(fmt::format("Connected signal \"{}\" has more then 1 demention. This is not allowed.", + sigCtx.inputPort.getSignal().getGlobalId())); + } + if (auto sampleType = signal.getDescriptor().getSampleType(); allowedSampleTypes.find(sampleType) == allowedSampleTypes.cend()) + { + status.addError(fmt::format("Connected signal \"{}\" has an incompatible sample type ({}).", + sigCtx.inputPort.getSignal().getGlobalId(), + convertSampleTypeToString(sampleType))); + } + } + status.merge(HandlerBase::validateSignalContexts(signalContexts)); + return status; +} + +MqttData RawHandler::processSignalContext(SignalContext& signalContext) +{ + MqttData messages; + const auto conn = signalContext.inputPort.getConnection(); + if (!conn.assigned()) + return messages; + + PacketPtr packet = conn.dequeue(); + while (packet.assigned()) + { + if (packet.getType() == PacketType::Event) + { + auto eventPacket = packet.asPtr(true); + LOG_T("Processing {} event", eventPacket.getEventId()) + if (eventPacket.getEventId() == event_packet_id::DATA_DESCRIPTOR_CHANGED) + { + DataDescriptorPtr valueSignalDescriptor = eventPacket.getParameters().get(event_packet_param::DATA_DESCRIPTOR); + DataDescriptorPtr domainSignalDescriptor = eventPacket.getParameters().get(event_packet_param::DOMAIN_DATA_DESCRIPTOR); + processSignalDescriptorChanged(signalContext, valueSignalDescriptor, domainSignalDescriptor); + messages.needRevalidation = true; + break; + } + } + else if (packet.getType() == PacketType::Data) + { + auto dataPacket = packet.asPtr(); + for (size_t i = 0; i < dataPacket.getSampleCount(); ++i) + messages.data.emplace_back(processDataPacket(signalContext, dataPacket, i)); + } + + packet = conn.dequeue(); + } + return messages; +} + +void RawHandler::processSignalDescriptorChanged(SignalContext&, + const DataDescriptorPtr&, + const DataDescriptorPtr&) +{ +} + +std::string RawHandler::buildTopicName(const SignalContext& signalContext) +{ + return signalContext.inputPort.getSignal().getGlobalId().toStdString(); +} + +MqttDataSamplePtr RawHandler::processDataPacket(SignalContext& signalContext, const DataPacketPtr& dataPacket, size_t offset) +{ + (void)signalContext; + auto msg = toDataBuffer(dataPacket, offset); + std::string topic = buildTopicName(signalContext); + return std::make_shared>>(signalContext.previewSignal, std::move(topic), std::move(msg)); +} + +template +std::vector copyData(daq::DataPacketPtr packet, size_t offset) +{ + std::vector data; + auto size = sizeof(T); + data.resize(size); + memcpy(data.data(), (static_cast(packet.getData()) + offset), size); + return data; +} + +std::vector RawHandler::toDataBuffer(daq::DataPacketPtr packet, size_t offset) +{ + std::vector data; + + switch (packet.getDataDescriptor().getSampleType()) + { + case SampleType::Float64: + data = copyData::Type>(packet, offset); + break; + case SampleType::Float32: + data = copyData::Type>(packet, offset); + break; + case SampleType::UInt64: + data = copyData::Type>(packet, offset); + break; + case SampleType::Int64: + data = copyData::Type>(packet, offset); + break; + case SampleType::UInt32: + data = copyData::Type>(packet, offset); + break; + case SampleType::Int32: + data = copyData::Type>(packet, offset); + break; + case SampleType::UInt16: + data = copyData::Type>(packet, offset); + break; + case SampleType::Int16: + data = copyData::Type>(packet, offset); + break; + case SampleType::UInt8: + data = copyData::Type>(packet, offset); + break; + case SampleType::Int8: + data = copyData::Type>(packet, offset); + break; + case SampleType::String: + case SampleType::Binary: + { + data.resize(packet.getDataSize()); + memcpy(data.data(), packet.getData(), packet.getDataSize()); + } + break; + default: + break; + } + return data; +} + +ListPtr RawHandler::getTopics(const std::vector& signalContexts) +{ + auto res = List(); + for (const auto& sigCtx : signalContexts) + { + if (!sigCtx.inputPort.getConnection().assigned()) + continue; + auto t = buildTopicName(sigCtx); + res.pushBack(String(t)); + } + return res; +} + +std::string RawHandler::getSchema() +{ + return std::string("Raw data"); +}; + +END_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE diff --git a/modules/mqtt_streaming_module/src/signal_arr_atomic_sample_handler.cpp b/modules/mqtt_streaming_module/src/signal_arr_atomic_sample_handler.cpp index 6ee7d6cc..51c2516c 100644 --- a/modules/mqtt_streaming_module/src/signal_arr_atomic_sample_handler.cpp +++ b/modules/mqtt_streaming_module/src/signal_arr_atomic_sample_handler.cpp @@ -10,8 +10,9 @@ BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE SignalArrayAtomicSampleHandler::SignalArrayAtomicSampleHandler(WeakRefPtr parentFb, SignalValueJSONKey signalNamesMode, std::string topic) - : HandlerBase(parentFb, signalNamesMode), - topic(topic) + : HandlerBase(parentFb), + topic(topic), + signalNamesMode(signalNamesMode) { } From 4ec94f98d0532c932f80bdd3a129f4eb66885913 Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Mon, 9 Mar 2026 14:53:12 +0100 Subject: [PATCH 04/15] new properties for raw publishing mode --- .../include/mqtt_streaming_module/constants.h | 1 + .../mqtt_streaming_module/handler_factory.h | 34 +++++++----- .../include/mqtt_streaming_module/types.h | 7 +++ .../src/mqtt_publisher_fb_impl.cpp | 52 ++++++++++++++----- 4 files changed, 68 insertions(+), 26 deletions(-) diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/constants.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/constants.h index c241b8e9..9d60f492 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/constants.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/constants.h @@ -40,6 +40,7 @@ static constexpr const char* PROPERTY_NAME_DEC_VALUE_NAME = "ValueKey"; static constexpr const char* PROPERTY_NAME_DEC_TS_NAME = "DomainKey"; static constexpr const char* PROPERTY_NAME_DEC_UNIT = "Unit"; +static constexpr const char* PROPERTY_NAME_PUB_MODE = "Mode"; static constexpr const char* PROPERTY_NAME_PUB_TOPIC_MODE = "TopicMode"; static constexpr const char* PROPERTY_NAME_PUB_TOPIC_NAME = "Topic"; static constexpr const char* PROPERTY_NAME_PUB_GROUP_VALUES = "GroupValues"; diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/handler_factory.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/handler_factory.h index af790b7b..54e62ff2 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/handler_factory.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/handler_factory.h @@ -22,6 +22,7 @@ #include #include #include +#include BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE @@ -31,22 +32,31 @@ class HandlerFactory static std::unique_ptr create(WeakRefPtr parentFb, const PublisherFbConfig config, const std::string& publisherFbGlobalId) { - if (config.topicMode == TopicMode::Single) + if (config.mode == PublisherMode::Raw) { - const auto topic = config.topicName.empty() ? publisherFbGlobalId : config.topicName; - if (config.groupValues) - return std::make_unique(parentFb, config.valueFieldName, topic, config.groupValuesPackSize); - else - return std::make_unique(parentFb, config.valueFieldName, topic); + return std::make_unique(parentFb); } - else if (config.topicMode == TopicMode::PerSignal) + else { - if (config.groupValues) - return std::make_unique(parentFb, config.valueFieldName, config.groupValuesPackSize); - else - return std::make_unique(parentFb, config.valueFieldName); + if (config.topicMode == TopicMode::Single) + { + const auto topic = config.topicName.empty() ? publisherFbGlobalId : config.topicName; + if (config.groupValues) + return std::make_unique(parentFb, + config.valueFieldName, + topic, + config.groupValuesPackSize); + else + return std::make_unique(parentFb, config.valueFieldName, topic); + } + else if (config.topicMode == TopicMode::PerSignal) + { + if (config.groupValues) + return std::make_unique(parentFb, config.valueFieldName, config.groupValuesPackSize); + else + return std::make_unique(parentFb, config.valueFieldName); + } } - return std::make_unique(parentFb, config.valueFieldName); } }; diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/types.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/types.h index d416a50d..7a20835d 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/types.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/types.h @@ -72,6 +72,12 @@ struct MqttData { } }; +enum class PublisherMode { + Json = 0, + Raw, + _count +}; + enum class TopicMode { PerSignal = 0, Single, @@ -86,6 +92,7 @@ enum class SignalValueJSONKey { }; struct PublisherFbConfig { + PublisherMode mode; TopicMode topicMode; std::string topicName; bool groupValues; diff --git a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp index 920de0cb..0c075e7d 100644 --- a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp @@ -86,13 +86,22 @@ void MqttPublisherFbImpl::removed() FunctionBlockTypePtr MqttPublisherFbImpl::CreateType() { auto defaultConfig = PropertyObject(); + + { + auto builder = SelectionPropertyBuilder(PROPERTY_NAME_PUB_MODE, List("JSON", "Raw"), 0) + .setDescription("Selects the mode of publishing. In JSON mode, the function block converts signal samples into " + "JSON messages and publishes them to MQTT topics. In Raw mode, the function block publishes raw " + "signal samples to MQTT topics without any conversion. By default it is set to JSON mode."); + defaultConfig.addProperty(builder.build()); + } { auto builder = SelectionPropertyBuilder(PROPERTY_NAME_PUB_TOPIC_MODE, List("TopicPerSignal", "SingleTopic"), 0) .setDescription( "Selects whether to publish all signals to separate MQTT topics (one per signal, TopicPerSignal mode) or to a single " "topic (SingleTopic mode), one for all signals. Choose 0 for TopicPerSignal mode, 1 for SingleTopic mode. By " - "default it is set to TopicPerSignal mode."); + "default it is set to TopicPerSignal mode.") + .setVisible(EvalValue(std::string("$") + PROPERTY_NAME_PUB_MODE + " == 0")); defaultConfig.addProperty(builder.build()); } { @@ -100,25 +109,25 @@ FunctionBlockTypePtr MqttPublisherFbImpl::CreateType() StringPropertyBuilder(PROPERTY_NAME_PUB_TOPIC_NAME, "") .setDescription( "Topic name for publishing in SingleTopic mode. If left empty, the Publisher's Global ID is used as the topic name.") - .setVisible(EvalValue(std::string("$") + PROPERTY_NAME_PUB_TOPIC_MODE + " == 1")); + .setVisible(EvalValue(std::string("($") + PROPERTY_NAME_PUB_TOPIC_MODE + " == 1) && ($" + PROPERTY_NAME_PUB_MODE + " == 0)")); defaultConfig.addProperty(builder.build()); } { auto builder = SelectionPropertyBuilder(PROPERTY_NAME_PUB_VALUE_FIELD_NAME, List("GlobalID", "LocalID", "Name"), 0) - .setDescription("Describes how to name a JSON value field. By default it is set to GlobalID."); + .setDescription("Describes how to name a JSON value field. By default it is set to GlobalID.") + .setVisible(EvalValue(std::string("$") + PROPERTY_NAME_PUB_MODE + " == 0")); defaultConfig.addProperty(builder.build()); } { - auto builder = - BoolPropertyBuilder(PROPERTY_NAME_PUB_GROUP_VALUES, False) - .setDescription( - "Enables the use of a sample pack for a signal. By default it is set to false."); + auto builder = BoolPropertyBuilder(PROPERTY_NAME_PUB_GROUP_VALUES, False) + .setDescription("Enables the use of a sample pack for a signal. By default it is set to false.") + .setVisible(EvalValue(std::string("$") + PROPERTY_NAME_PUB_MODE + " == 0")); defaultConfig.addProperty(builder.build()); } { auto builder = IntPropertyBuilder(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE, DEFAULT_PUB_PACK_SIZE) .setMinValue(1) - .setVisible(EvalValue(std::string("($") + PROPERTY_NAME_PUB_GROUP_VALUES + ")")) + .setVisible(EvalValue(std::string("($") + PROPERTY_NAME_PUB_GROUP_VALUES + ") && ($" + PROPERTY_NAME_PUB_MODE + " == 0)")) .setDescription(fmt::format("Set the size of the sample pack when publishing grouped values. " "By default it is set to {}.", DEFAULT_PUB_PACK_SIZE)); @@ -135,8 +144,9 @@ FunctionBlockTypePtr MqttPublisherFbImpl::CreateType() } { auto builder = BoolPropertyBuilder(PROPERTY_NAME_PUB_PREVIEW_SIGNAL, False) - .setDescription("Enables previewing of the publishing data in the function block's output signal. " - "By default it is set to false."); + .setVisible(EvalValue(std::string("$") + PROPERTY_NAME_PUB_MODE + " == 0")) + .setDescription("Enables previewing of the publishing data in the function block's output signal. " + "By default it is set to false."); defaultConfig.addProperty(builder.build()); } { @@ -423,7 +433,7 @@ void MqttPublisherFbImpl::initProperties(const PropertyObjectPtr& config) { auto builder = ListPropertyBuilder(PROPERTY_NAME_PUB_TOPICS, List()) .setReadOnly(true) - .setVisible(EvalValue(std::string("$") + PROPERTY_NAME_PUB_TOPIC_MODE + " == 0")) + .setVisible(EvalValue(std::string("($") + PROPERTY_NAME_PUB_TOPIC_MODE + " == 0) || ($" + PROPERTY_NAME_PUB_MODE + " == 1)")) .setDescription("List of currently used MQTT topics for publishing in TopicPerSignal mode."); objPtr.addProperty(builder.build()); @@ -440,6 +450,7 @@ void MqttPublisherFbImpl::initProperties(const PropertyObjectPtr& config) void MqttPublisherFbImpl::readProperties() { + int tmpMode = readProperty(PROPERTY_NAME_PUB_MODE, 0); int tmpTopicMode = readProperty(PROPERTY_NAME_PUB_TOPIC_MODE, 0); config.groupValues = readProperty(PROPERTY_NAME_PUB_GROUP_VALUES, false); @@ -448,7 +459,7 @@ void MqttPublisherFbImpl::readProperties() config.qos = readProperty(PROPERTY_NAME_PUB_QOS, DEFAULT_PUB_QOS); config.periodMs = readProperty(PROPERTY_NAME_PUB_READ_PERIOD, DEFAULT_PUB_READ_PERIOD); config.topicName = readProperty(PROPERTY_NAME_PUB_TOPIC_NAME, globalId.toStdString()); - config.enablePreview = readProperty(PROPERTY_NAME_PUB_PREVIEW_SIGNAL, false); + settingErrors.clear(); hasSettingError = false; @@ -463,6 +474,19 @@ void MqttPublisherFbImpl::readProperties() settingErrors.push_back(fmt::format("{} property has invalid value.", PROPERTY_NAME_PUB_VALUE_FIELD_NAME)); } + if (tmpMode < static_cast(PublisherMode::_count) && tmpMode >= 0) + { + config.mode = static_cast(tmpMode); + } + else + { + config.mode = PublisherMode::Json; + hasSettingError = true; + settingErrors.push_back("Mode has invalid value."); + } + + config.enablePreview = (config.mode != PublisherMode::Raw) && readProperty(PROPERTY_NAME_PUB_PREVIEW_SIGNAL, false); + if (tmpTopicMode < static_cast(TopicMode::_count) && tmpTopicMode >= 0) { config.topicMode = static_cast(tmpTopicMode); @@ -477,14 +501,14 @@ void MqttPublisherFbImpl::readProperties() if (config.topicName.empty()) { config.topicName = globalId.toStdString(); - hasEmptyTopic = (config.topicMode == TopicMode::Single); + hasEmptyTopic = (config.mode == PublisherMode::Json && config.topicMode == TopicMode::Single); } else { hasEmptyTopic = false; } - if (config.topicMode == TopicMode::Single) + if (config.mode == PublisherMode::Json && config.topicMode == TopicMode::Single) { auto result = mqtt::MqttDataWrapper::validateTopic(config.topicName, loggerComponent); hasSettingError = !result.success; From b860d69a4dc8fa929b666d0166481331f1c80c15 Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Mon, 9 Mar 2026 14:53:52 +0100 Subject: [PATCH 05/15] tests --- .../mqtt_streaming_module/raw_handler.h | 14 ++++ .../mqtt_streaming_module/src/raw_handler.cpp | 13 ---- .../tests/test_mqtt_publisher_fb.cpp | 75 +++++++++++++++---- 3 files changed, 74 insertions(+), 28 deletions(-) diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/raw_handler.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/raw_handler.h index 71272edc..f7d24b29 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/raw_handler.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/raw_handler.h @@ -18,6 +18,7 @@ #include #include +#include BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE @@ -41,6 +42,19 @@ class RawHandler : public HandlerBase MqttDataSamplePtr processDataPacket(SignalContext& signalContext, const DataPacketPtr& dataPacket, size_t offset); std::string buildTopicName(const SignalContext& signalContext); std::vector toDataBuffer(daq::DataPacketPtr packet, size_t offset); + + inline static const std::set allowedSampleTypes{SampleType::Float64, + SampleType::Float32, + SampleType::UInt8, + SampleType::Int8, + SampleType::UInt16, + SampleType::Int16, + SampleType::UInt32, + SampleType::Int32, + SampleType::UInt64, + SampleType::Int64, + SampleType::Binary, + SampleType::String}; }; END_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE diff --git a/modules/mqtt_streaming_module/src/raw_handler.cpp b/modules/mqtt_streaming_module/src/raw_handler.cpp index 3d1e1d91..7cc063a6 100644 --- a/modules/mqtt_streaming_module/src/raw_handler.cpp +++ b/modules/mqtt_streaming_module/src/raw_handler.cpp @@ -4,7 +4,6 @@ #include #include #include -#include BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE @@ -26,18 +25,6 @@ MqttData RawHandler::processSignalContexts(std::vector& signalCon ProcedureStatus RawHandler::validateSignalContexts(const std::vector& signalContexts) const { - static const std::set allowedSampleTypes{SampleType::Float64, - SampleType::Float32, - SampleType::UInt8, - SampleType::Int8, - SampleType::UInt16, - SampleType::Int16, - SampleType::UInt32, - SampleType::Int32, - SampleType::UInt64, - SampleType::Int64, - SampleType::Binary, - SampleType::String}; ProcedureStatus status{true, {}}; for (const auto& sigCtx : signalContexts) { diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp index 4a06134f..c339e266 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp @@ -591,41 +591,52 @@ TEST_F(MqttPublisherFbTest, DefaultConfig) ASSERT_TRUE(defaultConfig.assigned()); - ASSERT_EQ(defaultConfig.getAllProperties().getCount(), 8u); + EXPECT_EQ(defaultConfig.getAllProperties().getCount(), 9u); + + ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_PUB_MODE)); + ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_PUB_MODE).getValueType(), CoreType::ctInt); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_MODE).asPtr(), 0u); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_MODE).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_PUB_TOPIC_MODE)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_MODE).getValueType(), CoreType::ctInt); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE).asPtr(), 0u); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE).asPtr(), 0u); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_MODE).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_PUB_GROUP_VALUES)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_PUB_GROUP_VALUES).getValueType(), CoreType::ctBool); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_GROUP_VALUES).asPtr(), False); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_GROUP_VALUES).asPtr(), False); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_GROUP_VALUES).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_PUB_VALUE_FIELD_NAME)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_PUB_VALUE_FIELD_NAME).getValueType(), CoreType::ctInt); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_VALUE_FIELD_NAME).asPtr(), 0u); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_VALUE_FIELD_NAME).asPtr(), 0u); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_VALUE_FIELD_NAME).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE).getValueType(), CoreType::ctInt); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE).asPtr(), DEFAULT_PUB_PACK_SIZE); - ASSERT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE).getVisible()); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE).asPtr(), DEFAULT_PUB_PACK_SIZE); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_PUB_QOS)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_PUB_QOS).getValueType(), CoreType::ctInt); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_QOS).asPtr(), DEFAULT_PUB_QOS); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_QOS).asPtr(), DEFAULT_PUB_QOS); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_QOS).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_PUB_READ_PERIOD)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_PUB_READ_PERIOD).getValueType(), CoreType::ctInt); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_READ_PERIOD).asPtr(), DEFAULT_PUB_READ_PERIOD); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_READ_PERIOD).asPtr(), DEFAULT_PUB_READ_PERIOD); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_READ_PERIOD).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_PUB_TOPIC_NAME)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_NAME).getValueType(), CoreType::ctString); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_TOPIC_NAME).asPtr().toStdString(), ""); - ASSERT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_NAME).getVisible()); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_TOPIC_NAME).asPtr().toStdString(), ""); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_NAME).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_PUB_PREVIEW_SIGNAL)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_PUB_PREVIEW_SIGNAL).getValueType(), CoreType::ctBool); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_PREVIEW_SIGNAL).asPtr().getValue(False), False); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_PUB_PREVIEW_SIGNAL).asPtr().getValue(False), False); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_PREVIEW_SIGNAL).getVisible()); } TEST_F(MqttPublisherFbTest, PropertyVisibility) @@ -634,10 +645,42 @@ TEST_F(MqttPublisherFbTest, PropertyVisibility) daq::FunctionBlockTypePtr fbt = MqttPublisherFbImpl::CreateType(); daq::PropertyObjectPtr defaultConfig = fbt.createDefaultConfig(); - defaultConfig.setPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE, 0); // Set to Single topic - ASSERT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_NAME).getVisible()); - defaultConfig.setPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE, 1); // Set to Multi topic - ASSERT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_NAME).getVisible()); + { + defaultConfig.setPropertyValue(PROPERTY_NAME_PUB_MODE, 0); // Set JSON mode + defaultConfig.setPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE, 0); // Set to Single topic + ASSERT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_NAME).getVisible()); + defaultConfig.setPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE, 1); // Set to Multi topic + ASSERT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_NAME).getVisible()); + } + + { + defaultConfig.setPropertyValue(PROPERTY_NAME_PUB_MODE, 0); // Set JSON mode + defaultConfig.setPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE, 0); // Set to Single topic + + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_MODE).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_MODE).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_GROUP_VALUES).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_VALUE_FIELD_NAME).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_QOS).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_READ_PERIOD).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_NAME).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_PREVIEW_SIGNAL).getVisible()); + } + + { + defaultConfig.setPropertyValue(PROPERTY_NAME_PUB_MODE, 1); // Set Raw mode + + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_MODE).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_MODE).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_GROUP_VALUES).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_VALUE_FIELD_NAME).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_QOS).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_PUB_READ_PERIOD).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_TOPIC_NAME).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_PUB_PREVIEW_SIGNAL).getVisible()); + } } TEST_F(MqttPublisherFbTest, Config) @@ -645,6 +688,7 @@ TEST_F(MqttPublisherFbTest, Config) StartUp(); auto config = clientMqttFb.getAvailableFunctionBlockTypes().get(PUB_FB_NAME).createDefaultConfig(); + config.setPropertyValue(PROPERTY_NAME_PUB_MODE, 0); config.setPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE, 1); config.setPropertyValue(PROPERTY_NAME_PUB_GROUP_VALUES, True); config.setPropertyValue(PROPERTY_NAME_PUB_VALUE_FIELD_NAME, 1); @@ -1019,6 +1063,7 @@ TEST_F(MqttPublisherFbTest, WrongConfig) StartUp(); daq::FunctionBlockPtr fb; auto config = clientMqttFb.getAvailableFunctionBlockTypes().get(PUB_FB_NAME).createDefaultConfig(); + config.setPropertyValue(PROPERTY_NAME_PUB_MODE, 0); config.setPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE, 1); config.setPropertyValue(PROPERTY_NAME_PUB_TOPIC_NAME, String("/test/#")); ASSERT_NO_THROW(fb = clientMqttFb.addFunctionBlock(PUB_FB_NAME, config)); From 32786ef6cc70fb7f4b4477ac5afc1749c3370bc8 Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Mon, 9 Mar 2026 19:10:39 +0100 Subject: [PATCH 06/15] tests --- .../tests/test_mqtt_publisher_fb.cpp | 206 +++++++++++++++++- .../tests/MqttAsyncClientWrapper.cpp | 31 ++- .../tests/MqttAsyncClientWrapper.h | 34 +++ 3 files changed, 268 insertions(+), 3 deletions(-) diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp index c339e266..960fbce6 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp @@ -1,13 +1,14 @@ #include "MqttAsyncClientWrapper.h" #include "mqtt_streaming_helper/timer.h" #include "mqtt_streaming_module/mqtt_publisher_fb_impl.h" +#include "opendaq/binary_data_packet_factory.h" #include "test_daq_test_helper.h" #include #include +#include #include #include #include -#include using namespace daq; using namespace daq::modules::mqtt_streaming_module; @@ -251,6 +252,7 @@ class MqttPublisherFbHelper : public DaqTestHelper uint32_t readPeriod = 20) { auto config = clientMqttFb.getAvailableFunctionBlockTypes().get(PUB_FB_NAME).createDefaultConfig(); + config.setPropertyValue(PROPERTY_NAME_PUB_MODE, 0); config.setPropertyValue(PROPERTY_NAME_PUB_TOPIC_MODE, multiTopic ? 1 : 0); config.setPropertyValue(PROPERTY_NAME_PUB_GROUP_VALUES, groupV ? True : False); config.setPropertyValue(PROPERTY_NAME_PUB_VALUE_FIELD_NAME, useSignalNames ? 2 : 0); @@ -262,6 +264,17 @@ class MqttPublisherFbHelper : public DaqTestHelper fb = clientMqttFb.addFunctionBlock(PUB_FB_NAME, config); } + void CreateRawPublisherFB(int qos = 2, + uint32_t readPeriod = 20) + { + auto config = clientMqttFb.getAvailableFunctionBlockTypes().get(PUB_FB_NAME).createDefaultConfig(); + config.setPropertyValue(PROPERTY_NAME_PUB_MODE, 1); + config.setPropertyValue(PROPERTY_NAME_PUB_QOS, qos); + config.setPropertyValue(PROPERTY_NAME_PUB_READ_PERIOD, readPeriod); + config.setPropertyValue(PROPERTY_NAME_PUB_PREVIEW_SIGNAL, False); + fb = clientMqttFb.addFunctionBlock(PUB_FB_NAME, config); + } + bool CreateSubscriber(std::string postfix = "_subscriberId") { subscriber = std::make_unique(buildClientId(postfix)); @@ -521,6 +534,30 @@ class MqttPublisherFbHelper : public DaqTestHelper return ok; } + template + bool transfer(const std::string& topic, + SignalHelper& helper, + const std::vector>& data) + { + constexpr uint32_t DELAY_BETWEEN_PACKS= 20; + constexpr uint32_t BASE_TIMEOUT = 5000; + std::promise receivedPromise; + auto receivedFuture = receivedPromise.get_future(); + std::atomic done{false}; + subscriber->expectRawMsgs(topic, data, receivedPromise, done); + + helper::utils::Timer receiveTimer(BASE_TIMEOUT + data.size() * DELAY_BETWEEN_PACKS); + bool ok = subscriber->subscribe(topic, 2); + if (!ok) + return false; + auto future = helper.send(data, DELAY_BETWEEN_PACKS); + auto status = receivedFuture.wait_for(receiveTimer.remain()); + future.wait(); + subscriber = nullptr; + ok = (status == std::future_status::ready) && receivedFuture.get(); + return ok; + } + std::string buildTopicName(const std::string& postfix = "") { return std::string("test/topic/publisherFB/") + std::string(::testing::UnitTest::GetInstance()->current_test_info()->name()) + postfix; @@ -1397,6 +1434,173 @@ TEST_P(MqttPublisherFbPTest, TransferSharedTsArr) param); } +TEST_P(MqttPublisherFbPTest, TransferRaw) +{ + constexpr size_t sampleCnt = 15; + H param = GetParam(); + std::visit( + [&](auto& templateParam) + { + SignalHelper> help{}; + StartUp(); + ASSERT_NO_THROW(CreateRawPublisherFB()); + + fb.getInputPorts()[0].connect(help.signal0); + + ASSERT_TRUE(CreateSubscriber()); + + const auto data = help.generateTestData(sampleCnt); + const std::string topic = help.signal0.getGlobalId().toStdString(); + auto ok = transfer(topic, help, data); + ASSERT_TRUE(ok); + ASSERT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), + Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager())); + ASSERT_EQ(fb.getStatusContainer().getStatus(MQTT_PUB_FB_SET_STATUS_NAME), + EnumerationWithIntValue(MQTT_PUB_FB_SET_STATUS_TYPE, + static_cast(MqttPublisherFbImpl::SettingStatus::Valid), + daqInstance.getContext().getTypeManager())); + ASSERT_EQ(fb.getStatusContainer().getStatus(MQTT_PUB_FB_PUB_STATUS_NAME), + EnumerationWithIntValue(MQTT_PUB_FB_PUB_STATUS_TYPE, + static_cast(MqttPublisherFbImpl::PublishingStatus::Ok), + daqInstance.getContext().getTypeManager())); + }, + param); +} + +TEST_F(MqttPublisherFbTest, TransferRawString) +{ + constexpr size_t sampleCnt = 15; + + SignalHelper help{}; + StartUp(); + ASSERT_NO_THROW(CreateRawPublisherFB()); + + fb.getInputPorts()[0].connect(help.signal0); + + ASSERT_TRUE(CreateSubscriber()); + + const auto messages = std::vector{{"String 0"}, {"Long string 1 with a message that exceeds the usual length"}, {"Просто строка"}, {"S"}}; + std::vector> data; + for (const auto& msg : messages) + { + data.emplace_back(msg.cbegin(), msg.cend()); + } + const std::string topic = help.signal0.getGlobalId().toStdString(); + + constexpr uint32_t DELAY_BETWEEN_PACKS= 20; + constexpr uint32_t BASE_TIMEOUT = 5000; + std::promise receivedPromise; + auto receivedFuture = receivedPromise.get_future(); + std::atomic done{false}; + subscriber->expectRawMsgsBinaryData(topic, data, receivedPromise, done); + + bool ok = subscriber->subscribe(topic, 2); + ASSERT_TRUE(ok); + + auto future = std::async(std::launch::async, + [&messages, delay = DELAY_BETWEEN_PACKS, &help]() + { + for (size_t i = 0; i < messages.size(); ++i) + { + if (i != 0 && delay != 0) + std::this_thread::sleep_for(std::chrono::milliseconds(delay)); + + auto domainPacket = DataPacket(help.signal0.getDomainSignal().getDescriptor(), 1, i); + uint64_t ts = DataPacket(help.signal0.getDomainSignal().getDescriptor(), 1, i).getLastValue(); + *(reinterpret_cast(domainPacket.getData())) = ts; + + auto dataPacket = BinaryDataPacket(domainPacket, help.signal0.getDescriptor(), messages[i].size()); + std::memcpy(dataPacket.getRawData(), messages[i].data(), messages[i].size()); + help.signal0.sendPacket(std::move(dataPacket)); + } + }); + helper::utils::Timer receiveTimer(BASE_TIMEOUT + data.size() * DELAY_BETWEEN_PACKS); + auto status = receivedFuture.wait_for(receiveTimer.remain()); + future.wait(); + subscriber = nullptr; + + ASSERT_TRUE(status == std::future_status::ready); + ASSERT_TRUE(receivedFuture.get()); + ASSERT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), + Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager())); + ASSERT_EQ(fb.getStatusContainer().getStatus(MQTT_PUB_FB_SET_STATUS_NAME), + EnumerationWithIntValue(MQTT_PUB_FB_SET_STATUS_TYPE, + static_cast(MqttPublisherFbImpl::SettingStatus::Valid), + daqInstance.getContext().getTypeManager())); + ASSERT_EQ(fb.getStatusContainer().getStatus(MQTT_PUB_FB_PUB_STATUS_NAME), + EnumerationWithIntValue(MQTT_PUB_FB_PUB_STATUS_TYPE, + static_cast(MqttPublisherFbImpl::PublishingStatus::Ok), + daqInstance.getContext().getTypeManager())); + +} + + +TEST_F(MqttPublisherFbTest, TransferRawBinaryData) +{ + constexpr size_t sampleCnt = 15; + + SignalHelper help{}; + StartUp(); + ASSERT_NO_THROW(CreateRawPublisherFB()); + + fb.getInputPorts()[0].connect(help.signal0); + + ASSERT_TRUE(CreateSubscriber()); + + const auto data = std::vector>{std::vector{0x01, 0x02, 0x03, 0x04, 0x05}, + std::vector{0x11, 0x12, 0x13, 0x14}, + std::vector{0x21, 0x22, 0x23, 0x24, 0x25, 0x26}, + std::vector{0x31}, + std::vector{0x41, 0x42, 0x43, 0x44, 0x45}}; + const std::string topic = help.signal0.getGlobalId().toStdString(); + + constexpr uint32_t DELAY_BETWEEN_PACKS= 20; + constexpr uint32_t BASE_TIMEOUT = 5000; + std::promise receivedPromise; + auto receivedFuture = receivedPromise.get_future(); + std::atomic done{false}; + subscriber->expectRawMsgsBinaryData(topic, data, receivedPromise, done); + + bool ok = subscriber->subscribe(topic, 2); + ASSERT_TRUE(ok); + + auto future = std::async(std::launch::async, + [&data, delay = DELAY_BETWEEN_PACKS, &help]() + { + for (size_t i = 0; i < data.size(); ++i) + { + if (i != 0 && delay != 0) + std::this_thread::sleep_for(std::chrono::milliseconds(delay)); + + auto domainPacket = DataPacket(help.signal0.getDomainSignal().getDescriptor(), 1, i); + uint64_t ts = DataPacket(help.signal0.getDomainSignal().getDescriptor(), 1, i).getLastValue(); + *(reinterpret_cast(domainPacket.getData())) = ts; + + auto dataPacket = BinaryDataPacket(domainPacket, help.signal0.getDescriptor(), data[i].size()); + std::memcpy(dataPacket.getRawData(), data[i].data(), data[i].size()); + help.signal0.sendPacket(std::move(dataPacket)); + } + }); + helper::utils::Timer receiveTimer(BASE_TIMEOUT + data.size() * DELAY_BETWEEN_PACKS); + auto status = receivedFuture.wait_for(receiveTimer.remain()); + future.wait(); + subscriber = nullptr; + + ASSERT_TRUE(status == std::future_status::ready); + ASSERT_TRUE(receivedFuture.get()); + ASSERT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), + Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager())); + ASSERT_EQ(fb.getStatusContainer().getStatus(MQTT_PUB_FB_SET_STATUS_NAME), + EnumerationWithIntValue(MQTT_PUB_FB_SET_STATUS_TYPE, + static_cast(MqttPublisherFbImpl::SettingStatus::Valid), + daqInstance.getContext().getTypeManager())); + ASSERT_EQ(fb.getStatusContainer().getStatus(MQTT_PUB_FB_PUB_STATUS_NAME), + EnumerationWithIntValue(MQTT_PUB_FB_PUB_STATUS_TYPE, + static_cast(MqttPublisherFbImpl::PublishingStatus::Ok), + daqInstance.getContext().getTypeManager())); + +} + TEST_P(MqttPublisherFbPTest, DISABLED_TransferMultimessage) { constexpr size_t sampleCnt = 15; diff --git a/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.cpp b/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.cpp index dd3cc6ec..8a80b317 100644 --- a/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.cpp +++ b/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.cpp @@ -2,8 +2,7 @@ #include "MqttAsyncClient.h" #include "mqtt_streaming_helper/timer.h" #include -#include -#include + using namespace std::chrono; using namespace helper::utils; @@ -191,3 +190,31 @@ void MqttAsyncClientWrapper::expectMultiMsgs(const std::string& topic, promise.set_value(true); }); } + +void MqttAsyncClientWrapper::expectRawMsgsBinaryData(const std::string& topic, + const std::vector>& data, + std::promise& promise, + std::atomic& done) +{ + instance->setMessageArrivedCb(topic, + [topic, &done, &data, &promise, i = size_t(0)](const mqtt::MqttAsyncClient&, + mqtt::MqttMessage& receivedMsg) mutable + { + if (receivedMsg.getTopic() != topic || i >= data.size()) + return; + if (data[i].size() != receivedMsg.getData().size()) + { + std::cerr << "Sample size and packet data size are not the same!" << std::endl; + return; + } + if (receivedMsg.getData() != data[i]) + { + std::cerr << "Data value and packet value are not the same!" << std::endl; + return; + } + + bool expected = false; + if (++i == data.size() && done.compare_exchange_strong(expected, true)) + promise.set_value(true); + }); +} diff --git a/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.h b/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.h index 418440f8..85579f7c 100644 --- a/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.h +++ b/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.h @@ -1,6 +1,7 @@ #pragma once #include "MqttAsyncClient.h" #include +#include class MqttAsyncClientWrapper { @@ -18,6 +19,39 @@ class MqttAsyncClientWrapper bool subscribe(const std::string& topic, int qos); void expectMsgs(const std::string& topic, const std::vector& msgs, std::promise& promise, std::atomic& done); void expectMultiMsgs(const std::string& topic, const std::vector& msgs, std::promise& promise, std::atomic& done); + void expectRawMsgsBinaryData(const std::string& topic, + const std::vector>& data, + std::promise& promise, + std::atomic& done); + + template + void expectRawMsgs(const std::string& topic, + const std::vector>& data, + std::promise& promise, + std::atomic& done) + { + instance->setMessageArrivedCb(topic, + [topic, &done, &data, &promise, i = size_t(0)](const mqtt::MqttAsyncClient&, + mqtt::MqttMessage& receivedMsg) mutable + { + if (receivedMsg.getTopic() != topic || i >= data.size()) + return; + if (sizeof(T) != receivedMsg.getData().size()) + { + std::cerr << "Sample size and packet data size are not the same!" << std::endl; + return; + } + if (*(reinterpret_cast(receivedMsg.getData().data())) != data[i].first) + { + std::cerr << "Data value and packet value are not the same!" << std::endl; + return; + } + + bool expected = false; + if (++i == data.size() && done.compare_exchange_strong(expected, true)) + promise.set_value(true); + }); + } std::unique_ptr instance; std::promise connectedPromise; From 86f1b9d13ae5477b03d4efae042cc0d849cd3268 Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Wed, 11 Mar 2026 15:49:31 +0100 Subject: [PATCH 07/15] subscriber and decoder FBs: new way to set data timestamps --- .../include/mqtt_streaming_module/constants.h | 2 + .../mqtt_json_decoder_fb_impl.h | 7 +- .../mqtt_subscriber_fb_impl.h | 16 ++ .../mqtt_streaming_module/property_helper.h | 41 ++++ .../mqtt_streaming_module/src/CMakeLists.txt | 2 + .../src/mqtt_json_decoder_fb_impl.cpp | 81 ++++--- .../src/mqtt_subscriber_fb_impl.cpp | 199 +++++++++++++----- .../tests/test_mqtt_json_decoder_fb.cpp | 50 ++++- .../tests/test_mqtt_subscriber_fb.cpp | 28 ++- .../include/MqttDataWrapper.h | 15 +- .../src/MqttDataWrapper.cpp | 62 ++++-- 11 files changed, 389 insertions(+), 114 deletions(-) create mode 100644 modules/mqtt_streaming_module/include/mqtt_streaming_module/property_helper.h diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/constants.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/constants.h index 9d60f492..ac20c1ff 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/constants.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/constants.h @@ -34,9 +34,11 @@ static constexpr const char* PROPERTY_NAME_SUB_JSON_CONFIG_FILE = "JSONConfigFil static constexpr const char* PROPERTY_NAME_SUB_QOS = "QoS"; static constexpr const char* PROPERTY_NAME_SUB_TOPIC = "Topic"; static constexpr const char* PROPERTY_NAME_SUB_PREVIEW_SIGNAL = "EnablePreviewSignal"; +static constexpr const char* PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE = "DomainMode"; static constexpr const char* PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING = "MessageIsString"; static constexpr const char* PROPERTY_NAME_DEC_VALUE_NAME = "ValueKey"; +static constexpr const char* PROPERTY_NAME_DEC_TS_MODE = "DomainMode"; static constexpr const char* PROPERTY_NAME_DEC_TS_NAME = "DomainKey"; static constexpr const char* PROPERTY_NAME_DEC_UNIT = "Unit"; diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h index b3f684d1..3dde5f11 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h @@ -38,12 +38,13 @@ class MqttJsonDecoderFbImpl final : public FunctionBlock const FunctionBlockTypePtr& type, const PropertyObjectPtr& config = nullptr); - static FunctionBlockTypePtr CreateType(); - void processMessage(const std::string& json); + DAQ_MQTT_STREAM_MODULE_API static FunctionBlockTypePtr CreateType(); + void processMessage(const std::string& json, const uint64_t externalTs); protected: struct FbConfig { std::string valueFieldName; + mqtt::MqttDataWrapper::DomainSignalMode tsMode; std::string tsFieldName; std::string unitSymbol; }; @@ -69,8 +70,6 @@ class MqttJsonDecoderFbImpl final : public FunctionBlock void initProperties(const PropertyObjectPtr& config); void readProperties(); - template - retT readProperty(const std::string& propertyName, const retT defaultValue); void propertyChanged(); void updateStatuses(); diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h index 75704325..409aa6a8 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h @@ -44,6 +44,13 @@ class MqttSubscriberFbImpl final : public FunctionBlock } }; + enum class DomainSignalMode : EnumType + { + None = 0, + SystemTime, + _count + }; + explicit DAQ_MQTT_STREAM_MODULE_API MqttSubscriberFbImpl(const ContextPtr& ctx, const ComponentPtr& parent, const FunctionBlockTypePtr& type, @@ -65,9 +72,12 @@ class MqttSubscriberFbImpl final : public FunctionBlock DictObjectPtr nestedFbTypes; std::vector nestedFunctionBlocks; SignalConfigPtr outputSignal; + SignalConfigPtr outputDomainSignal; bool enablePreview; + DomainSignalMode previewDomainMode; bool previewIsString; std::atomic waitingForData; + uint64_t lastTsValue; DAQ_MQTT_STREAM_MODULE_API void onSignalsMessage(const mqtt::MqttAsyncClient& subscriber, const mqtt::MqttMessage& msg); @@ -76,8 +86,14 @@ class MqttSubscriberFbImpl final : public FunctionBlock void initNestedFbTypes(); void createSignals(); + SignalConfigPtr createDomainSignal(); + void removePreviewSignal(); + void removeDomainSignal(); + void reconfigureSignal(); void clearSubscribedTopic(); + DataPacketPtr createDomainDataPacket(const uint64_t epochTime); + void processMessage(const mqtt::MqttMessage& msg); void initProperties(const PropertyObjectPtr& config); void readProperties(); diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/property_helper.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/property_helper.h new file mode 100644 index 00000000..5ed424e0 --- /dev/null +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/property_helper.h @@ -0,0 +1,41 @@ +/* + * Copyright 2022-2025 openDAQ d.o.o. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#pragma once +#include +#include + +BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE + +namespace property_helper +{ + +template +retT readProperty(const PropertyObjectPtr objPtr, const std::string& propertyName, const retT defaultValue) +{ + retT returnValue{defaultValue}; + if (objPtr.hasProperty(propertyName)) + { + auto property = objPtr.getPropertyValue(propertyName).asPtrOrNull(); + if (property.assigned()) + { + returnValue = property.getValue(defaultValue); + } + } + return returnValue; +} +} // namespace property_helper +END_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE diff --git a/modules/mqtt_streaming_module/src/CMakeLists.txt b/modules/mqtt_streaming_module/src/CMakeLists.txt index 88a48d73..9d833ef8 100644 --- a/modules/mqtt_streaming_module/src/CMakeLists.txt +++ b/modules/mqtt_streaming_module/src/CMakeLists.txt @@ -20,6 +20,7 @@ set(SRC_Include common.h raw_handler.h types.h status_helper.h + property_helper.h status_adaptor.h ) @@ -45,6 +46,7 @@ source_group("common" FILES ${MODULE_HEADERS_DIR}/common.h ${MODULE_HEADERS_DIR}/helper.h ${MODULE_HEADERS_DIR}/types.h ${MODULE_HEADERS_DIR}/status_helper.h + ${MODULE_HEADERS_DIR}/property_helper.h ${MODULE_HEADERS_DIR}/status_adaptor.h helper.cpp ) diff --git a/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp index 63d131e5..cb2f7abc 100644 --- a/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp @@ -1,5 +1,6 @@ #include "mqtt_streaming_module/constants.h" #include +#include #include #include @@ -25,6 +26,7 @@ MqttJsonDecoderFbImpl::MqttJsonDecoderFbImpl(const ContextPtr& ctx, FunctionBlockTypePtr MqttJsonDecoderFbImpl::CreateType() { + using DSM = mqtt::MqttDataWrapper::DomainSignalMode; auto defaultConfig = PropertyObject(); { auto builder = @@ -34,9 +36,24 @@ FunctionBlockTypePtr MqttJsonDecoderFbImpl::CreateType() defaultConfig.addProperty(builder.build()); } + { + auto builder = + SelectionPropertyBuilder(PROPERTY_NAME_DEC_TS_MODE, + List("None", "Extract from message", "System time"), + static_cast(DSM::None)) + .setDescription( + "Defines how the timestamp of the decoded signal is generated. By default it is set to None, which means that " + "the decoded signal doesn't have a timestamp. If set to Extract from message, the JSON decoder will try " + "to extract the timestamp from the incoming JSON messages. If set to System time, the timestamp of the decoded signal " + "is set to the system time when the JSON message is received."); + defaultConfig.addProperty(builder.build()); + } + { auto builder = StringPropertyBuilder(PROPERTY_NAME_DEC_TS_NAME, String("")) + .setVisible(EvalValue(std::string("$") + PROPERTY_NAME_DEC_TS_MODE + + " == " + std::to_string(static_cast(DSM::ExtractFromMessage)))) .setDescription( "Specifies the JSON field name from which timestamp will be extracted. This property is " "optional. If it is set it should be contained in the incoming JSON messages. Otherwise, a parsing error will occur."); @@ -87,43 +104,50 @@ void MqttJsonDecoderFbImpl::initProperties(const PropertyObjectPtr& config) void MqttJsonDecoderFbImpl::readProperties() { + using namespace property_helper; + using DSM = mqtt::MqttDataWrapper::DomainSignalMode; auto lock = this->getRecursiveConfigLock(); configValid = true; configMsg.clear(); - config.valueFieldName = readProperty(PROPERTY_NAME_DEC_VALUE_NAME, ""); + config.valueFieldName = readProperty(objPtr, PROPERTY_NAME_DEC_VALUE_NAME, ""); if (config.valueFieldName.empty()) { configMsg = fmt::format("\"{}\" property is empty!", PROPERTY_NAME_DEC_VALUE_NAME); configValid = false; } - config.tsFieldName = readProperty(PROPERTY_NAME_DEC_TS_NAME, ""); - config.unitSymbol = readProperty(PROPERTY_NAME_DEC_UNIT, ""); - jsonDataWorker.setValueFieldName(config.valueFieldName); - jsonDataWorker.setTimestampFieldName(config.tsFieldName); - waitingData = configValid.load(); - updateStatuses(); -} + config.tsFieldName = readProperty(objPtr, PROPERTY_NAME_DEC_TS_NAME, ""); + config.unitSymbol = readProperty(objPtr, PROPERTY_NAME_DEC_UNIT, ""); + auto tmpTsMode = readProperty(objPtr, PROPERTY_NAME_DEC_TS_MODE, static_cast(DSM::None)); -template -retT MqttJsonDecoderFbImpl::readProperty(const std::string& propertyName, const retT defaultValue) -{ - retT returnValue{}; - bool isPresent = false; - if (objPtr.hasProperty(propertyName)) + if (tmpTsMode < static_cast(DSM::_count) && tmpTsMode >= 0) { - auto property = objPtr.getPropertyValue(propertyName).asPtrOrNull(); - if (property.assigned()) + config.tsMode = static_cast(tmpTsMode); + } + else + { + configMsg = fmt::format("Wrong value for the \"{}\" property!", PROPERTY_NAME_DEC_TS_MODE); + configValid = false; + config.tsMode = DSM::None; + } + if (config.tsMode == DSM::ExtractFromMessage) + { + if (config.tsFieldName.empty()) { - isPresent = true; - returnValue = property.getValue(defaultValue); + configMsg = + fmt::format("Empty \"{}\" property is not allowed for the \'Extract from message\' mode", PROPERTY_NAME_DEC_TS_NAME); + configValid = false; } } - if (!isPresent) + else { - LOG_W("{} property is missing! Default value is set (\"{}\")", propertyName, defaultValue); + config.tsFieldName = ""; } - return returnValue; + jsonDataWorker.setValueFieldName(config.valueFieldName); + jsonDataWorker.setTimestampFieldName(config.tsFieldName); + jsonDataWorker.setDomainSignalMode(config.tsMode); + waitingData = configValid.load(); + updateStatuses(); } void MqttJsonDecoderFbImpl::propertyChanged() @@ -154,13 +178,13 @@ void MqttJsonDecoderFbImpl::updateStatuses() } } -void MqttJsonDecoderFbImpl::processMessage(const std::string& json) +void MqttJsonDecoderFbImpl::processMessage(const std::string& json, const uint64_t externalTs) { if (configValid) { auto lock = this->getRecursiveConfigLock(); waitingData = false; - auto status = jsonDataWorker.createAndSendDataPacket(json); + auto status = jsonDataWorker.createAndSendDataPacket(json, externalTs); parsingSucceeded = status.success; if (status.success) { @@ -186,7 +210,7 @@ void MqttJsonDecoderFbImpl::createSignal() outputSignal = createAndAddSignal(DEFAULT_VALUE_SIGNAL_LOCAL_ID, dataDescBdr.build()); outputSignal.setName(config.valueFieldName); - if (config.tsFieldName != "") + if (config.tsMode != mqtt::MqttDataWrapper::DomainSignalMode::None) { outputSignal.setDomainSignal(createDomainSignal()); } @@ -210,19 +234,20 @@ void MqttJsonDecoderFbImpl::reconfigureSignal(const FbConfig& prevConfig) outputSignal.setDescriptor(descBilder.build()); } - if (prevConfig.tsFieldName != config.tsFieldName) + if (prevConfig.tsMode != config.tsMode) { - if (prevConfig.tsFieldName.empty() && !config.tsFieldName.empty()) + using DSM = mqtt::MqttDataWrapper::DomainSignalMode; + if (prevConfig.tsMode == DSM::None && config.tsMode != DSM::None) { outputSignal.setDomainSignal(createDomainSignal()); } - else if (!prevConfig.tsFieldName.empty() && config.tsFieldName.empty()) + else if (prevConfig.tsMode != DSM::None && config.tsMode == DSM::None) { outputSignal.setDomainSignal(nullptr); removeSignal(outputDomainSignal); outputDomainSignal = nullptr; } - else if (!prevConfig.tsFieldName.empty() && !config.tsFieldName.empty()) + else { // do nothing } diff --git a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp index ad285134..25f6d227 100644 --- a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp @@ -1,4 +1,5 @@ #include "mqtt_streaming_module/constants.h" +#include #include #include #include @@ -24,8 +25,10 @@ MqttSubscriberFbImpl::MqttSubscriberFbImpl(const ContextPtr& ctx, topicForSubscribing(""), nestedFbTypes(nullptr), enablePreview(false), + previewDomainMode(DomainSignalMode::None), previewIsString(false), - waitingForData(false) + waitingForData(false), + lastTsValue(0) { initComponentStatus(); initNestedFbTypes(); @@ -37,6 +40,13 @@ MqttSubscriberFbImpl::MqttSubscriberFbImpl(const ContextPtr& ctx, { readJsonConfig(); } + else + { + LOG_W("\'{}\' property is set. JSON configuration (\'{}\' and \'{}\') will be ignored.", + PROPERTY_NAME_SUB_TOPIC, + PROPERTY_NAME_SUB_JSON_CONFIG, + PROPERTY_NAME_SUB_JSON_CONFIG_FILE); + } createSignals(); subscribeToTopic(); } @@ -107,7 +117,7 @@ FunctionBlockTypePtr MqttSubscriberFbImpl::CreateType() } { auto builder = - SelectionPropertyBuilder(PROPERTY_NAME_PUB_QOS, List(0, 1, 2), DEFAULT_PUB_QOS) + SelectionPropertyBuilder(PROPERTY_NAME_SUB_QOS, List(0, 1, 2), DEFAULT_SUB_QOS) .setDescription( fmt::format("MQTT Quality of Service level for subscribing. It can be 0 (at most once), 1 (at least once), or 2 " "(exactly once). By default it is set to {}.", @@ -120,6 +130,18 @@ FunctionBlockTypePtr MqttSubscriberFbImpl::CreateType() "By default it is set to false."); defaultConfig.addProperty(builder.build()); } + { + auto builder = + SelectionPropertyBuilder(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE, + List("None", "System time"), + static_cast(DomainSignalMode::None)) + .setVisible(EvalValue(std::string("$") + PROPERTY_NAME_SUB_PREVIEW_SIGNAL)) + .setDescription( + "Defines the domain of the preview signal. By default it is set to None, which means that the preview signal doesn't " + "have a timestamp. If set to System time, the preview signal's timestamp is set to the system time when the MQTT " + "message is received."); + defaultConfig.addProperty(builder.build()); + } { auto builder = BoolPropertyBuilder(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING, False) .setVisible(EvalValue(std::string("$") + PROPERTY_NAME_SUB_PREVIEW_SIGNAL)) @@ -164,42 +186,24 @@ void MqttSubscriberFbImpl::initNestedFbTypes() void MqttSubscriberFbImpl::readProperties() { + using namespace property_helper; auto lock = this->getRecursiveConfigLock(); topicForSubscribing.clear(); - std::string topic; - if (objPtr.hasProperty(PROPERTY_NAME_SUB_TOPIC)) - { - auto topicStr = objPtr.getPropertyValue(PROPERTY_NAME_SUB_TOPIC).asPtrOrNull(); - if (topicStr.assigned()) - topic = topicStr.toStdString(); - } + std::string topic = readProperty(objPtr, PROPERTY_NAME_SUB_TOPIC, std::string("")); setTopic(topic); - if (objPtr.hasProperty(PROPERTY_NAME_SUB_QOS)) + qos = readProperty(objPtr, PROPERTY_NAME_SUB_QOS, DEFAULT_SUB_QOS); + enablePreview = readProperty(objPtr, PROPERTY_NAME_SUB_PREVIEW_SIGNAL, false); + previewIsString = readProperty(objPtr, PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING, false); + auto tmpPreviewDomain = + readProperty(objPtr, PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE, static_cast(DomainSignalMode::None)); + if (tmpPreviewDomain < static_cast(DomainSignalMode::_count) && tmpPreviewDomain >= 0) { - auto qosProp = objPtr.getPropertyValue(PROPERTY_NAME_SUB_QOS).asPtrOrNull(); - if (qosProp.assigned()) - { - const uint32_t qos = qosProp.getValue(DEFAULT_SUB_QOS); - this->qos = (qos > 2) ? DEFAULT_SUB_QOS : qos; - } - } - if (objPtr.hasProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL)) - { - auto previewProp = objPtr.getPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL).asPtrOrNull(); - if (previewProp.assigned()) - { - this->enablePreview = previewProp.getValue(False); - } + previewDomainMode = static_cast(tmpPreviewDomain); } - - if (objPtr.hasProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING)) + else { - auto isStringProp = objPtr.getPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING).asPtrOrNull(); - if (isStringProp.assigned()) - { - this->previewIsString = isStringProp.getValue(False); - } + previewDomainMode = DomainSignalMode::None; } } @@ -306,27 +310,7 @@ void MqttSubscriberFbImpl::propertyChanged() return; } readProperties(); - if (enablePreview) - { - if (!outputSignal.assigned()) - { - createSignals(); - } - else if ((outputSignal.getDescriptor().getSampleType() == SampleType::String) != previewIsString) - { - outputSignal.setDescriptor(DataDescriptorBuilderCopy(outputSignal.getDescriptor()) - .setSampleType(previewIsString ? SampleType::String : SampleType::Binary) - .build()); - } - } - else - { - if (outputSignal.assigned()) - { - removeSignal(outputSignal); - outputSignal = nullptr; - } - } + reconfigureSignal(); result = subscribeToTopic(); } @@ -410,10 +394,16 @@ void MqttSubscriberFbImpl::processMessage(const mqtt::MqttMessage& msg) std::string jsonObjStr(msg.getData().begin(), msg.getData().end()); auto acqlock = this->getAcquisitionLock2(); + using namespace std::chrono; + const uint64_t epochTime = duration_cast(system_clock::now().time_since_epoch()).count(); + daq::DataPacketPtr domainPacket; if (enablePreview) { - const auto outputPacket = BinaryDataPacket(nullptr, outputSignal.getDescriptor(), msg.getData().size()); + domainPacket = createDomainDataPacket(epochTime); + const auto outputPacket = BinaryDataPacket(domainPacket, outputSignal.getDescriptor(), msg.getData().size()); memcpy(outputPacket.getData(), msg.getData().data(), msg.getData().size()); + if (outputDomainSignal.assigned() && domainPacket.assigned()) + outputDomainSignal.sendPacket(domainPacket); outputSignal.sendPacket(outputPacket); } @@ -422,12 +412,31 @@ void MqttSubscriberFbImpl::processMessage(const mqtt::MqttMessage& msg) if (fb.assigned()) { auto decoderFb = reinterpret_cast(*fb); - decoderFb->processMessage(jsonObjStr); + decoderFb->processMessage(jsonObjStr, epochTime); } } } } +DataPacketPtr MqttSubscriberFbImpl::createDomainDataPacket(const uint64_t epochTime) +{ + DataPacketPtr domainPacket; + if (!outputDomainSignal.assigned()) + return domainPacket; + if (previewDomainMode == DomainSignalMode::SystemTime) + { + domainPacket = daq::DataPacket(outputDomainSignal.getDescriptor(), 1); + std::memcpy(domainPacket.getRawData(), &epochTime, sizeof(epochTime)); + if (lastTsValue == epochTime) + { + if (statusContainer.getStatus("ComponentStatus") != ComponentStatus::Error) + setComponentStatusWithMessage(ComponentStatus::Warning, "Domain signal value is the same as previous. Data may be lost"); + } + lastTsValue = epochTime; + } + return domainPacket; +} + void MqttSubscriberFbImpl::createSignals() { auto lock = this->getRecursiveConfigLock(); // ??? @@ -435,6 +444,88 @@ void MqttSubscriberFbImpl::createSignals() { const auto signalDsc = DataDescriptorBuilder().setSampleType(previewIsString ? SampleType::String : SampleType::Binary).build(); outputSignal = createAndAddSignal(DEFAULT_VALUE_SIGNAL_LOCAL_ID, signalDsc); + if (previewDomainMode != DomainSignalMode::None) + { + outputSignal.setDomainSignal(createDomainSignal()); + } + } +} + +SignalConfigPtr MqttSubscriberFbImpl::createDomainSignal() +{ + auto getEpoch = []() -> std::string + { + const std::time_t epochTime = std::chrono::system_clock::to_time_t(std::chrono::time_point{}); + char buf[48]; + strftime(buf, sizeof buf, "%Y-%m-%dT%H:%M:%SZ", gmtime(&epochTime)); + return {buf}; + }; + + const auto domainSignalDsc = DataDescriptorBuilder() + .setSampleType(SampleType::UInt64) + .setUnit(Unit("s", -1, "seconds", "time")) + .setRule(ExplicitDomainDataRule()) + .setTickResolution(Ratio(1, 1'000'000)) + .setOrigin(getEpoch()) + .setName("Time") + .build(); + outputDomainSignal = createAndAddSignal(DEFAULT_TS_SIGNAL_LOCAL_ID, domainSignalDsc, false); + return outputDomainSignal; +} + +void MqttSubscriberFbImpl::removePreviewSignal() +{ + if (outputSignal.assigned()) + { + removeDomainSignal(); + removeSignal(outputSignal); + outputSignal = nullptr; + } +} + +void MqttSubscriberFbImpl::removeDomainSignal() +{ + if (outputSignal.assigned()) + outputSignal.setDomainSignal(nullptr); + + if (outputDomainSignal.assigned()) + { + removeSignal(outputDomainSignal); + outputDomainSignal = nullptr; + } +} + +void MqttSubscriberFbImpl::reconfigureSignal() +{ + auto lock = this->getRecursiveConfigLock(); + if (enablePreview) + { + if (!outputSignal.assigned()) + { + createSignals(); + } + else + { + if ((outputSignal.getDescriptor().getSampleType() == SampleType::String) != previewIsString) + { + outputSignal.setDescriptor(DataDescriptorBuilderCopy(outputSignal.getDescriptor()) + .setSampleType(previewIsString ? SampleType::String : SampleType::Binary) + .build()); + } + if (previewDomainMode == DomainSignalMode::None) + { + removeDomainSignal(); + } + else if (!outputDomainSignal.assigned()) + { + createDomainSignal(); + outputSignal.setDomainSignal(outputDomainSignal); + } + } + } + else + { + removePreviewSignal(); } } diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp index 286f4927..728da659 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp @@ -383,6 +383,8 @@ class MqttJsonDecoderFbHelper : public DaqTestHelper daq::StringPtr typeId = daq::String(JSON_DECODER_FB_NAME); auto config = subMqttFb.getAvailableFunctionBlockTypes().get(JSON_DECODER_FB_NAME).createDefaultConfig(); + if (!tsF.empty()) + config.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, 1); config.setPropertyValue(PROPERTY_NAME_DEC_VALUE_NAME, valueF); config.setPropertyValue(PROPERTY_NAME_DEC_TS_NAME, tsF); config.setPropertyValue(PROPERTY_NAME_DEC_UNIT, unitSymbol); @@ -534,21 +536,61 @@ TEST_F(MqttJsonDecoderFbTest, DefaultConfig) ASSERT_TRUE(defaultConfig.assigned()); - ASSERT_EQ(defaultConfig.getAllProperties().getCount(), 3u); + EXPECT_EQ(defaultConfig.getAllProperties().getCount(), 4u); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_DEC_VALUE_NAME)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_DEC_VALUE_NAME).getValueType(), CoreType::ctString); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_DEC_VALUE_NAME).asPtr().getLength(), 0u); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_DEC_VALUE_NAME).asPtr().getLength(), 0u); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_VALUE_NAME).getVisible()); + + ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_DEC_TS_MODE)); + ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getValueType(), CoreType::ctInt); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_DEC_TS_MODE).asPtr(), static_cast(mqtt::MqttDataWrapper::DomainSignalMode::None)); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_DEC_TS_NAME)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_NAME).getValueType(), CoreType::ctString); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_DEC_TS_NAME).asPtr().getLength(), 0u); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_DEC_TS_NAME).asPtr().getLength(), 0u); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_NAME).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_DEC_UNIT)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_DEC_UNIT).getValueType(), CoreType::ctString); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_DEC_UNIT).asPtr().getLength(), 0u); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_DEC_UNIT).asPtr().getLength(), 0u); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_UNIT).getVisible()); } +TEST_F(MqttJsonDecoderFbTest, PropertyVisibility) +{ + daq::DictPtr fbTypes; + daq::FunctionBlockTypePtr fbt = MqttJsonDecoderFbImpl::CreateType(); + daq::PropertyObjectPtr defaultConfig = fbt.createDefaultConfig(); + + { + defaultConfig.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(mqtt::MqttDataWrapper::DomainSignalMode::None)); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_VALUE_NAME).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_NAME).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_UNIT).getVisible()); + } + + { + defaultConfig.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(mqtt::MqttDataWrapper::DomainSignalMode::ExtractFromMessage)); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_VALUE_NAME).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_NAME).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_UNIT).getVisible()); + } + + { + defaultConfig.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(mqtt::MqttDataWrapper::DomainSignalMode::ExternalTimestamp)); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_VALUE_NAME).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getVisible()); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_NAME).getVisible()); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_UNIT).getVisible()); + } +} + + TEST_F(MqttJsonDecoderFbTest, Config) { StartUp(); diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp index 142740be..5daac440 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp @@ -90,31 +90,42 @@ TEST_F(MqttSubscriberFbTest, DefaultConfig) ASSERT_TRUE(defaultConfig.assigned()); - EXPECT_EQ(defaultConfig.getAllProperties().getCount(), 6u); + EXPECT_EQ(defaultConfig.getAllProperties().getCount(), 7u); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_SUB_JSON_CONFIG)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_SUB_JSON_CONFIG).getValueType(), CoreType::ctString); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_JSON_CONFIG).asPtr().getLength(), 0u); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_JSON_CONFIG).asPtr().getLength(), 0u); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_SUB_JSON_CONFIG).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_SUB_JSON_CONFIG_FILE)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_SUB_JSON_CONFIG_FILE).getValueType(), CoreType::ctString); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_JSON_CONFIG_FILE).asPtr().getLength(), 0u); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_JSON_CONFIG_FILE).asPtr().getLength(), 0u); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_SUB_JSON_CONFIG_FILE).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_SUB_TOPIC)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_SUB_TOPIC).getValueType(), CoreType::ctString); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_TOPIC).asPtr().getLength(), 0u); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_TOPIC).asPtr().getLength(), 0u); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_SUB_TOPIC).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_SUB_QOS)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_SUB_QOS).getValueType(), CoreType::ctInt); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_QOS).asPtr().getValue(DEFAULT_SUB_QOS), DEFAULT_SUB_QOS); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_QOS).asPtr().getValue(DEFAULT_SUB_QOS), DEFAULT_SUB_QOS); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_SUB_QOS).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL).getValueType(), CoreType::ctBool); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL).asPtr().getValue(False), False); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL).asPtr().getValue(False), False); + EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL).getVisible()); + + ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE)); + ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE).getValueType(), CoreType::ctInt); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE).asPtr().getValue(0), 0); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING).getValueType(), CoreType::ctBool); - ASSERT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING).asPtr().getValue(False), False); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING).asPtr().getValue(False), False); + EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING).getVisible()); } TEST_F(MqttSubscriberFbTest, PropertyVisibility) @@ -123,10 +134,13 @@ TEST_F(MqttSubscriberFbTest, PropertyVisibility) daq::FunctionBlockTypePtr fbt = MqttSubscriberFbImpl::CreateType(); daq::PropertyObjectPtr defaultConfig = fbt.createDefaultConfig(); + defaultConfig.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, True); ASSERT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING).getVisible()); + ASSERT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE).getVisible()); defaultConfig.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, False); ASSERT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING).getVisible()); + ASSERT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE).getVisible()); } TEST_F(MqttSubscriberFbTest, Config) diff --git a/shared/mqtt_streaming_protocol/include/MqttDataWrapper.h b/shared/mqtt_streaming_protocol/include/MqttDataWrapper.h index 54aaef7d..7a1dbbea 100644 --- a/shared/mqtt_streaming_protocol/include/MqttDataWrapper.h +++ b/shared/mqtt_streaming_protocol/include/MqttDataWrapper.h @@ -97,6 +97,14 @@ class MqttDataWrapper final } }; + enum class DomainSignalMode : int + { + None = 0, + ExtractFromMessage, + ExternalTimestamp, + _count + }; + MqttDataWrapper(daq::LoggerComponentPtr loggerComponent); static CmdResult validateTopic(const daq::StringPtr topic, const daq::LoggerComponentPtr loggerComponent = nullptr); @@ -106,10 +114,11 @@ class MqttDataWrapper final std::string extractTopic(); CmdResult isJsonValid(); void setOutputSignal(daq::SignalConfigPtr outputSignal); - CmdResult createAndSendDataPacket(const std::string& json); + CmdResult createAndSendDataPacket(const std::string& json, const uint64_t externalTs); //bool hasDomainSignal(const SignalId& signalId) const; void setValueFieldName(std::string valueFieldName); void setTimestampFieldName(std::string tsFieldName); + void setDomainSignalMode(DomainSignalMode mode); private: rapidjson::Document doc; @@ -119,8 +128,10 @@ class MqttDataWrapper final daq::SignalConfigPtr outputSignal; // used for description how to extract data from sample json MqttMsgDescriptor msgDescriptor; + DomainSignalMode domainSignalType; - std::pair> extractDataSamples(const MqttMsgDescriptor& msgDescriptor, const std::string& json); + std::pair> + extractDataSamples(const std::string& json, const uint64_t externalTs); void sendDataSamples(const DataPackets& dataPackets); template DataPackets buildDataPackets(const T& value, uint64_t timestamp); diff --git a/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp b/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp index 5701dd44..bb718d16 100644 --- a/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp +++ b/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp @@ -110,7 +110,8 @@ namespace mqtt { MqttDataWrapper::MqttDataWrapper(daq::LoggerComponentPtr loggerComponent) - : loggerComponent(loggerComponent) + : loggerComponent(loggerComponent), + domainSignalType(DomainSignalMode::None) { } @@ -260,9 +261,9 @@ void MqttDataWrapper::setOutputSignal(daq::SignalConfigPtr outputSignal) this->outputSignal = outputSignal; } -MqttDataWrapper::CmdResult MqttDataWrapper::createAndSendDataPacket(const std::string& json) +MqttDataWrapper::CmdResult MqttDataWrapper::createAndSendDataPacket(const std::string& json, const uint64_t externalTs) { - auto [status, packets] = extractDataSamples(msgDescriptor, json); + auto [status, packets] = extractDataSamples(json, externalTs); if (status.success) { for (const auto& data : packets) @@ -283,8 +284,13 @@ void MqttDataWrapper::setTimestampFieldName(std::string tsFieldName) msgDescriptor.tsFieldName = std::move(tsFieldName); } -std::pair> MqttDataWrapper::extractDataSamples(const MqttMsgDescriptor& msgDescriptor, - const std::string& json) +void MqttDataWrapper::setDomainSignalMode(DomainSignalMode mode) +{ + domainSignalType = mode; +} + +std::pair> MqttDataWrapper::extractDataSamples(const std::string& json, + const uint64_t externalTs) { using ValueVariant = std::variant, std::vector, std::vector>; ValueVariant value{}; @@ -365,7 +371,8 @@ std::pair> MqttDataWrapper: } } } - else if (!msgDescriptor.tsFieldName.empty() && name == msgDescriptor.tsFieldName) + else if (domainSignalType == DomainSignalMode::ExtractFromMessage && !msgDescriptor.tsFieldName.empty() && + name == msgDescriptor.tsFieldName) { const auto& tsField = jsonDocument[name]; if (tsField.IsArray()) @@ -422,10 +429,6 @@ std::pair> MqttDataWrapper: } } } - else - { - LOG_T("Field \"{}\" is not supported.", name); - } } } } @@ -433,12 +436,12 @@ std::pair> MqttDataWrapper: { result.addError("Error deserializing MQTT payload. "); } - if (!hasValue || (!msgDescriptor.tsFieldName.empty() && !hasTS)) + if (!hasValue || (domainSignalType == DomainSignalMode::ExtractFromMessage && !msgDescriptor.tsFieldName.empty() && !hasTS)) { result.addError("Not all required fields are present in the JSON message. "); if (!hasValue) result.addError(fmt::format("Couldn't extract value field (\"{}\") from the JSON message. ", msgDescriptor.valueFieldName)); - if (!msgDescriptor.tsFieldName.empty() && !hasTS) + if (domainSignalType == DomainSignalMode::ExtractFromMessage && !msgDescriptor.tsFieldName.empty() && !hasTS) result.addError(fmt::format("Couldn't extract timestamp field (\"{}\") from the JSON message. ", msgDescriptor.tsFieldName)); } @@ -448,16 +451,33 @@ std::pair> MqttDataWrapper: [&](auto&& values) { using T = std::decay_t; - if (hasTS && ts.size() != values.size()) + if (domainSignalType == DomainSignalMode::ExtractFromMessage && hasTS && ts.size() != values.size()) { result.addError("Timestamp and value array sizes do not match. "); return; } + if (domainSignalType == DomainSignalMode::ExternalTimestamp && values.size() > 1) + { + result.addError("External timestamp mode doesn't support arrays of values. "); + return; + } if constexpr (std::is_same_v>) { for (size_t i = 0; i < values.size(); ++i) { - DataPackets dp = hasTS ? buildDataPackets(values[i], ts[i]) : buildDataPackets(values[i]); + DataPackets dp; + if (domainSignalType == DomainSignalMode::ExternalTimestamp) + { + dp = buildDataPackets(values[i], externalTs); + } + else if (domainSignalType == DomainSignalMode::ExtractFromMessage) + { + dp = buildDataPackets(values[i], ts[i]); + } + else if (domainSignalType == DomainSignalMode::None) + { + dp = buildDataPackets(values[i]); + } if (dp.dataPacket.assigned()) outputData.push_back(std::move(dp)); @@ -465,7 +485,19 @@ std::pair> MqttDataWrapper: } else { - DataPackets dp = hasTS ? buildDataPackets(values, ts) : buildDataPackets(values); + DataPackets dp; + if (domainSignalType == DomainSignalMode::ExternalTimestamp) + { + dp = buildDataPackets(values, externalTs); + } + else if (domainSignalType == DomainSignalMode::ExtractFromMessage) + { + dp = buildDataPackets(values, ts); + } + else if (domainSignalType == DomainSignalMode::None) + { + dp = buildDataPackets(values); + } if (dp.dataPacket.assigned()) outputData.push_back(std::move(dp)); From b671199a239a930b5a4703f3bf8b51e523925b4a Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Wed, 11 Mar 2026 15:50:49 +0100 Subject: [PATCH 08/15] redundant code removing --- .../mqtt_publisher_fb_impl.h | 2 -- .../src/mqtt_publisher_fb_impl.cpp | 35 ++++++------------- 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_publisher_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_publisher_fb_impl.h index 15313d42..13bf299b 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_publisher_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_publisher_fb_impl.h @@ -112,8 +112,6 @@ class MqttPublisherFbImpl final : public FunctionBlock void validateInputPorts(); void updateTopics(); void updateSchema(); - template - retT readProperty(const std::string& propertyName, const retT defaultValue); void runReaderThread(); void readerLoop(); void sendMessages(const MqttData& data); diff --git a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp index 0c075e7d..6a7b619a 100644 --- a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp @@ -5,6 +5,7 @@ #include #include #include +#include BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE @@ -450,15 +451,16 @@ void MqttPublisherFbImpl::initProperties(const PropertyObjectPtr& config) void MqttPublisherFbImpl::readProperties() { - int tmpMode = readProperty(PROPERTY_NAME_PUB_MODE, 0); - int tmpTopicMode = readProperty(PROPERTY_NAME_PUB_TOPIC_MODE, 0); + using namespace property_helper; + int tmpMode = readProperty(objPtr, PROPERTY_NAME_PUB_MODE, 0); + int tmpTopicMode = readProperty(objPtr, PROPERTY_NAME_PUB_TOPIC_MODE, 0); - config.groupValues = readProperty(PROPERTY_NAME_PUB_GROUP_VALUES, false); - int tmpValueFieldName = (readProperty(PROPERTY_NAME_PUB_VALUE_FIELD_NAME, 0)); - config.groupValuesPackSize = readProperty(PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE, DEFAULT_PUB_PACK_SIZE); - config.qos = readProperty(PROPERTY_NAME_PUB_QOS, DEFAULT_PUB_QOS); - config.periodMs = readProperty(PROPERTY_NAME_PUB_READ_PERIOD, DEFAULT_PUB_READ_PERIOD); - config.topicName = readProperty(PROPERTY_NAME_PUB_TOPIC_NAME, globalId.toStdString()); + config.groupValues = readProperty(objPtr, PROPERTY_NAME_PUB_GROUP_VALUES, false); + int tmpValueFieldName = (readProperty(objPtr, PROPERTY_NAME_PUB_VALUE_FIELD_NAME, 0)); + config.groupValuesPackSize = readProperty(objPtr, PROPERTY_NAME_PUB_GROUP_VALUES_PACK_SIZE, DEFAULT_PUB_PACK_SIZE); + config.qos = readProperty(objPtr, PROPERTY_NAME_PUB_QOS, DEFAULT_PUB_QOS); + config.periodMs = readProperty(objPtr, PROPERTY_NAME_PUB_READ_PERIOD, DEFAULT_PUB_READ_PERIOD); + config.topicName = readProperty(objPtr, PROPERTY_NAME_PUB_TOPIC_NAME, globalId.toStdString()); settingErrors.clear(); hasSettingError = false; @@ -485,7 +487,7 @@ void MqttPublisherFbImpl::readProperties() settingErrors.push_back("Mode has invalid value."); } - config.enablePreview = (config.mode != PublisherMode::Raw) && readProperty(PROPERTY_NAME_PUB_PREVIEW_SIGNAL, false); + config.enablePreview = (config.mode != PublisherMode::Raw) && readProperty(objPtr, PROPERTY_NAME_PUB_PREVIEW_SIGNAL, false); if (tmpTopicMode < static_cast(TopicMode::_count) && tmpTopicMode >= 0) { @@ -528,21 +530,6 @@ void MqttPublisherFbImpl::readProperties() } } -template -retT MqttPublisherFbImpl::readProperty(const std::string& propertyName, const retT defaultValue) -{ - retT returnValue{}; - if (objPtr.hasProperty(propertyName)) - { - auto property = objPtr.getPropertyValue(propertyName).asPtrOrNull(); - if (property.assigned()) - { - returnValue = property.getValue(defaultValue); - } - } - return returnValue; -} - void MqttPublisherFbImpl::runReaderThread() { LOGP_D("Using separate thread for rendering") From 5aa0b460e19e54daf0de81715c21faab4adeb1cc Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Wed, 11 Mar 2026 18:16:57 +0100 Subject: [PATCH 09/15] tests --- .../tests/test_mqtt_json_decoder_fb.cpp | 150 +++++++++++++----- .../tests/test_mqtt_subscriber_fb.cpp | 89 ++++++++--- 2 files changed, 178 insertions(+), 61 deletions(-) diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp index 728da659..2300ba95 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp @@ -15,6 +15,7 @@ using namespace daq; using namespace daq::modules::mqtt_streaming_module; +using DDSM = mqtt::MqttDataWrapper::DomainSignalMode; namespace { @@ -372,19 +373,18 @@ class MqttJsonDecoderFbHelper : public DaqTestHelper subMqttFb = new MqttSubscriberFbImpl(NullContext(), nullptr, fbType, nullptr, config); } - void CreateDecoderFB(std::string topic, std::string valueF, std::string tsF, std::string unitSymbol = "") + void CreateDecoderFB(std::string topic, std::string valueF, DDSM mode, std::string tsF, std::string unitSymbol = "") { CreateJsonFb(topic); - AddDecoderFb(valueF, tsF, unitSymbol); + AddDecoderFb(valueF, mode, tsF, unitSymbol); } - daq::FunctionBlockPtr AddDecoderFb(std::string valueF, std::string tsF, std::string unitSymbol = "") + daq::FunctionBlockPtr AddDecoderFb(std::string valueF, DDSM mode, std::string tsF, std::string unitSymbol = "") { daq::StringPtr typeId = daq::String(JSON_DECODER_FB_NAME); auto config = subMqttFb.getAvailableFunctionBlockTypes().get(JSON_DECODER_FB_NAME).createDefaultConfig(); - if (!tsF.empty()) - config.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, 1); + config.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(mode)); config.setPropertyValue(PROPERTY_NAME_DEC_VALUE_NAME, valueF); config.setPropertyValue(PROPERTY_NAME_DEC_TS_NAME, tsF); config.setPropertyValue(PROPERTY_NAME_DEC_UNIT, unitSymbol); @@ -426,6 +426,12 @@ class MqttJsonDecoderFbHelper : public DaqTestHelper return transferData(data, jsonDataTemplate); } + template + std::vector> transferDataWithSystemTime(const std::vector>& data, const std::string& jsonDataTemplate, std::vector& timePoints) + { + return transferDataWithSystemTime>(data, jsonDataTemplate, timePoints); + } + private: template std::vector transferData(const std::vector>& data, const std::string& jsonDataTemplate) { @@ -436,16 +442,46 @@ class MqttJsonDecoderFbHelper : public DaqTestHelper { tsF = extractFieldName(jsonDataTemplate, ""); } + DDSM mode = DDSM::None; + if (!tsF.empty()) + { + mode = DDSM::ExtractFromMessage; + } + CreateDecoderFB(topic, valueF, mode, tsF); + + auto signal = getSignals()[0]; + auto reader = daq::PacketReader(signal); + + auto msgs = replacePlaceholders(data, jsonDataTemplate); + for (const auto& str : msgs) + { + onSignalsMessage({topic, std::vector(str.begin(), str.end()), 1, 0}); + } - CreateDecoderFB(topic, valueF, tsF); + std::vector dataToReceive = read(reader, signal, 0); + return dataToReceive; + } + + template std::vector transferDataWithSystemTime(const std::vector>& data, const std::string& jsonDataTemplate, std::vector& timePoints) + { + const auto topic = buildTopicName(); + std::string valueF = extractFieldName(jsonDataTemplate, ""); + CreateDecoderFB(topic, valueF, DDSM::ExternalTimestamp, ""); auto signal = getSignals()[0]; auto reader = daq::PacketReader(signal); + auto getTime = []() + { + using namespace std::chrono; + return duration_cast(system_clock::now().time_since_epoch()).count(); + }; auto msgs = replacePlaceholders(data, jsonDataTemplate); + timePoints.push_back(getTime()); for (const auto& str : msgs) { onSignalsMessage({topic, std::vector(str.begin(), str.end()), 1, 0}); + timePoints.push_back(getTime()); } std::vector dataToReceive = read(reader, signal, 0); @@ -525,14 +561,7 @@ class MqttJsonFbUnitPTest : public ::testing::TestWithParam fbTypes; - daq::FunctionBlockTypePtr fbt; - daq::PropertyObjectPtr defaultConfig; - ASSERT_NO_THROW(fbTypes = subMqttFb.getAvailableFunctionBlockTypes()); - ASSERT_NO_THROW(fbt = fbTypes.get(JSON_DECODER_FB_NAME)); - ASSERT_NO_THROW(defaultConfig = fbt.createDefaultConfig()); + daq::PropertyObjectPtr defaultConfig = MqttJsonDecoderFbImpl::CreateType().createDefaultConfig(); ASSERT_TRUE(defaultConfig.assigned()); @@ -545,7 +574,7 @@ TEST_F(MqttJsonDecoderFbTest, DefaultConfig) ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_DEC_TS_MODE)); ASSERT_EQ(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getValueType(), CoreType::ctInt); - EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_DEC_TS_MODE).asPtr(), static_cast(mqtt::MqttDataWrapper::DomainSignalMode::None)); + EXPECT_EQ(defaultConfig.getPropertyValue(PROPERTY_NAME_DEC_TS_MODE).asPtr(), static_cast(DDSM::None)); EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getVisible()); ASSERT_TRUE(defaultConfig.hasProperty(PROPERTY_NAME_DEC_TS_NAME)); @@ -561,12 +590,10 @@ TEST_F(MqttJsonDecoderFbTest, DefaultConfig) TEST_F(MqttJsonDecoderFbTest, PropertyVisibility) { - daq::DictPtr fbTypes; - daq::FunctionBlockTypePtr fbt = MqttJsonDecoderFbImpl::CreateType(); - daq::PropertyObjectPtr defaultConfig = fbt.createDefaultConfig(); + daq::PropertyObjectPtr defaultConfig = MqttJsonDecoderFbImpl::CreateType().createDefaultConfig(); { - defaultConfig.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(mqtt::MqttDataWrapper::DomainSignalMode::None)); + defaultConfig.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(DDSM::None)); EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_VALUE_NAME).getVisible()); EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getVisible()); EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_NAME).getVisible()); @@ -574,7 +601,7 @@ TEST_F(MqttJsonDecoderFbTest, PropertyVisibility) } { - defaultConfig.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(mqtt::MqttDataWrapper::DomainSignalMode::ExtractFromMessage)); + defaultConfig.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(DDSM::ExtractFromMessage)); EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_VALUE_NAME).getVisible()); EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getVisible()); EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_NAME).getVisible()); @@ -582,7 +609,7 @@ TEST_F(MqttJsonDecoderFbTest, PropertyVisibility) } { - defaultConfig.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(mqtt::MqttDataWrapper::DomainSignalMode::ExternalTimestamp)); + defaultConfig.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(DDSM::ExternalTimestamp)); EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_VALUE_NAME).getVisible()); EXPECT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_MODE).getVisible()); EXPECT_FALSE(defaultConfig.getProperty(PROPERTY_NAME_DEC_TS_NAME).getVisible()); @@ -598,8 +625,10 @@ TEST_F(MqttJsonDecoderFbTest, Config) auto config = subMqttFb.getAvailableFunctionBlockTypes().get(JSON_DECODER_FB_NAME).createDefaultConfig(); config.setPropertyValue(PROPERTY_NAME_DEC_VALUE_NAME, "value"); + config.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(DDSM::ExtractFromMessage)); config.setPropertyValue(PROPERTY_NAME_DEC_TS_NAME, "timestamp"); config.setPropertyValue(PROPERTY_NAME_DEC_UNIT, "ppm"); + daq::FunctionBlockPtr fb; ASSERT_NO_THROW(fb = subMqttFb.addFunctionBlock(JSON_DECODER_FB_NAME, config)); EXPECT_EQ(fb.getSignals().getCount(), 1u); @@ -638,21 +667,49 @@ TEST_F(MqttJsonDecoderFbTest, CreationWithPartialConfig) auto config = PropertyObject(); config.addProperty(StringProperty(PROPERTY_NAME_DEC_VALUE_NAME, String("value"))); ASSERT_NO_THROW(fb = subMqttFb.addFunctionBlock(JSON_DECODER_FB_NAME, config)); + + EXPECT_EQ(fb.getSignals().getCount(), 1u); + EXPECT_EQ(fb.getSignals(search::Any()).getCount(), 1u); + EXPECT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), + Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager())); + EXPECT_NE(fb.getStatusContainer().getStatusMessage("ComponentStatus").toStdString().find("Waiting for data"), std::string::npos); + + fb.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(DDSM::ExtractFromMessage)); + EXPECT_EQ(fb.getSignals().getCount(), 1u); + EXPECT_EQ(fb.getSignals(search::Any()).getCount(), 2u); + EXPECT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), + Enumeration("ComponentStatusType", "Error", daqInstance.getContext().getTypeManager())); + EXPECT_NE(fb.getStatusContainer().getStatusMessage("ComponentStatus").toStdString().find("Configuration is invalid"), std::string::npos); + + fb.setPropertyValue(PROPERTY_NAME_DEC_TS_NAME, String("ts")); EXPECT_EQ(fb.getSignals().getCount(), 1u); - ASSERT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), + EXPECT_EQ(fb.getSignals(search::Any()).getCount(), 2u); + EXPECT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager())); EXPECT_NE(fb.getStatusContainer().getStatusMessage("ComponentStatus").toStdString().find("Waiting for data"), std::string::npos); + subMqttFb.removeFunctionBlock(fb); } { daq::FunctionBlockPtr fb; - auto config = PropertyObject(); - config.addProperty(StringProperty(PROPERTY_NAME_DEC_TS_NAME, String("ts"))); - ASSERT_NO_THROW(fb = subMqttFb.addFunctionBlock(JSON_DECODER_FB_NAME, config)); + ASSERT_NO_THROW(fb = subMqttFb.addFunctionBlock(JSON_DECODER_FB_NAME)); + + fb.setPropertyValue(PROPERTY_NAME_DEC_TS_MODE, static_cast(DDSM::ExtractFromMessage)); + fb.setPropertyValue(PROPERTY_NAME_DEC_TS_NAME, String("ts")); + EXPECT_EQ(fb.getSignals().getCount(), 1u); - ASSERT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), + EXPECT_EQ(fb.getSignals(search::Any()).getCount(), 2u); + EXPECT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), Enumeration("ComponentStatusType", "Error", daqInstance.getContext().getTypeManager())); EXPECT_NE(fb.getStatusContainer().getStatusMessage("ComponentStatus").toStdString().find("Configuration is invalid"), std::string::npos); + + fb.setPropertyValue(PROPERTY_NAME_DEC_VALUE_NAME, String("value")); + EXPECT_EQ(fb.getSignals().getCount(), 1u); + EXPECT_EQ(fb.getSignals(search::Any()).getCount(), 2u); + EXPECT_EQ(fb.getStatusContainer().getStatus("ComponentStatus"), + Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager())); + EXPECT_NE(fb.getStatusContainer().getStatusMessage("ComponentStatus").toStdString().find("Waiting for data"), std::string::npos); + subMqttFb.removeFunctionBlock(fb); } } @@ -679,6 +736,27 @@ TEST_P(MqttJsonFbDoubleDataPTest, DataTransferOneSignalDoubleWithoutDomain) EXPECT_NE(decoderObj.getStatusContainer().getStatusMessage("ComponentStatus").toStdString().find("Parsing succeeded"), std::string::npos); } +TEST_P(MqttJsonFbDoubleDataPTest, DataTransferOneSignalDoubleWithSystemTime) +{ + const auto dataToSend = GetParam(); + std::vector timePoints; + auto dataToReceive = transferDataWithSystemTime(dataToSend, VALID_JSON_DATA_1, timePoints); + ASSERT_EQ(dataToSend.size(), dataToReceive.size()); + ASSERT_TRUE(compareData(dataToSend, dataToReceive, false)); + + ASSERT_EQ(dataToSend.size() + 1, timePoints.size()); + + for (size_t i = 0; i < dataToReceive.size(); ++i) + { + EXPECT_GE(dataToReceive[i].second, timePoints[i]); + EXPECT_LE(dataToReceive[i].second, timePoints[i + 1]); + } + + ASSERT_EQ(decoderObj.getStatusContainer().getStatus("ComponentStatus"), + Enumeration("ComponentStatusType", "Ok", decoderObj.getContext().getTypeManager())); + EXPECT_NE(decoderObj.getStatusContainer().getStatusMessage("ComponentStatus").toStdString().find("Parsing succeeded"), std::string::npos); +} + INSTANTIATE_TEST_SUITE_P(DataTransferOneSignalDouble, MqttJsonFbDoubleDataPTest, ::testing::Values(DATA_DOUBLE_INT_0, DATA_DOUBLE_INT_1, DATA_DOUBLE_INT_2)); @@ -837,9 +915,9 @@ TEST_F(MqttJsonDecoderFbTest, DataTransferSeveralSignals) DaqInstanceInit(); auto clientFb0 = DaqAddClientMqttFb("127.0.0.1", DEFAULT_PORT); auto jsonFb0 = AddSubFb(topic); - auto decoderFb0 = AddDecoderFb(valueF0, tsF); - auto decoderFb1 = AddDecoderFb(valueF1, tsF); - auto decoderFb2 = AddDecoderFb(valueF2, ""); + auto decoderFb0 = AddDecoderFb(valueF0, DDSM::ExtractFromMessage, tsF); + auto decoderFb1 = AddDecoderFb(valueF1, DDSM::ExtractFromMessage, tsF); + auto decoderFb2 = AddDecoderFb(valueF2, DDSM::None, ""); auto signalList = List(); signalList.pushBack(decoderFb0.getSignals()[0]); @@ -894,7 +972,7 @@ TEST_F(MqttJsonDecoderFbTest, DataTransferMissingFieldOneSignal) const auto topic = buildTopicName(); std::string valueF = extractFieldName(VALID_JSON_DATA_1, ""); std::string tsF = "ts"; - CreateDecoderFB(topic, valueF, tsF); + CreateDecoderFB(topic, valueF, DDSM::ExtractFromMessage, tsF); auto signal = getSignals()[0]; auto reader = daq::PacketReader(signal); @@ -924,9 +1002,9 @@ TEST_F(MqttJsonDecoderFbTest, DataTransferMissingFieldSeveralSignals) DaqInstanceInit(); auto clientFb0 = DaqAddClientMqttFb("127.0.0.1", DEFAULT_PORT); auto jsonFb0 = AddSubFb(topic); - auto decoderFb0 = AddDecoderFb(valueF0, tsF); - auto decoderFb1 = AddDecoderFb(valueF1, tsF); - auto decoderFb2 = AddDecoderFb(valueF2, ""); + auto decoderFb0 = AddDecoderFb(valueF0, DDSM::ExtractFromMessage, tsF); + auto decoderFb1 = AddDecoderFb(valueF1, DDSM::ExtractFromMessage, tsF); + auto decoderFb2 = AddDecoderFb(valueF2, DDSM::None, ""); auto signalList = List(); signalList.pushBack(decoderFb0.getSignals()[0]); @@ -981,7 +1059,7 @@ TEST_F(MqttJsonFbCommunicationTest, FullDataTransfer) const std::string valueF = extractFieldName(msgTemplate, ""); const std::string tsF = extractFieldName(msgTemplate, ""); AddSubFb(topic); - AddDecoderFb(valueF, tsF); + AddDecoderFb(valueF, DDSM::ExtractFromMessage, tsF); const auto result = processTransfer("127.0.0.1", DEFAULT_PORT, topic, DATA_DOUBLE_INT_0, decoderObj.getSignals()[0]); @@ -1005,11 +1083,11 @@ TEST_F(MqttJsonFbCommunicationTest, DISABLED_FullDataTransferFor2MqttFbs) DaqInstanceInit(); auto clientFb0 = DaqAddClientMqttFb("127.0.0.1", 1883); auto jsonFb0 = AddSubFb(topic0); - auto decoderFb0 = AddDecoderFb(valueF, tsF); + auto decoderFb0 = AddDecoderFb(valueF, DDSM::ExtractFromMessage, tsF); auto clientFb1 = DaqAddClientMqttFb("127.0.0.1", 1884); auto jsonFb1 = AddSubFb(topic1); - auto decoderFb1 = AddDecoderFb(valueF, tsF); + auto decoderFb1 = AddDecoderFb(valueF, DDSM::ExtractFromMessage, tsF); const auto result0 = processTransfer("127.0.0.1", 1883, topic0, DATA_DOUBLE_INT_0, decoderFb0.getSignals()[0]); const auto result1 = processTransfer("127.0.0.1", 1884, topic1, DATA_DOUBLE_INT_1, decoderFb1.getSignals()[0]); diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp index 5daac440..cdd0107d 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp @@ -12,6 +12,7 @@ using namespace daq; using namespace daq::modules::mqtt_streaming_module; +using SDSM = MqttSubscriberFbImpl::DomainSignalMode; namespace daq::modules::mqtt_streaming_module { @@ -20,14 +21,21 @@ class MqttSubscriberFbHelper public: std::unique_ptr obj; - void CreateSubFB(std::string topic) + void CreateSubFB(std::string topic, + bool enablePreview = true, + SDSM previewTsMode = SDSM::None) { - auto config = PropertyObject(); - config.addProperty(StringProperty(PROPERTY_NAME_SUB_TOPIC, "")); - config.addProperty(BoolProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, False)); - const auto fbType = FunctionBlockType(SUB_FB_NAME, SUB_FB_NAME, "", config); + const auto fbType = MqttSubscriberFbImpl::CreateType(); + auto config = fbType.createDefaultConfig(); config.setPropertyValue(PROPERTY_NAME_SUB_TOPIC, topic); - config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, True); + config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, enablePreview ? True : False); + config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE, static_cast(previewTsMode)); + obj = std::make_unique(NullContext(), nullptr, fbType, nullptr, config); + } + + void CreateSubFB(daq::PropertyObjectPtr config) + { + const auto fbType = MqttSubscriberFbImpl::CreateType(); obj = std::make_unique(NullContext(), nullptr, fbType, nullptr, config); } @@ -80,13 +88,7 @@ class MqttSubscriberFbConfigFilePTest : public ::testing::TestWithParam fbTypes; - daq::FunctionBlockTypePtr fbt; - daq::PropertyObjectPtr defaultConfig; - ASSERT_NO_THROW(fbTypes = clientMqttFb.getAvailableFunctionBlockTypes()); - ASSERT_NO_THROW(fbt = fbTypes.get(SUB_FB_NAME)); - ASSERT_NO_THROW(defaultConfig = fbt.createDefaultConfig()); + daq::PropertyObjectPtr defaultConfig = MqttSubscriberFbImpl::CreateType().createDefaultConfig(); ASSERT_TRUE(defaultConfig.assigned()); @@ -130,10 +132,7 @@ TEST_F(MqttSubscriberFbTest, DefaultConfig) TEST_F(MqttSubscriberFbTest, PropertyVisibility) { - daq::DictPtr fbTypes; - daq::FunctionBlockTypePtr fbt = MqttSubscriberFbImpl::CreateType(); - daq::PropertyObjectPtr defaultConfig = fbt.createDefaultConfig(); - + daq::PropertyObjectPtr defaultConfig = MqttSubscriberFbImpl::CreateType().createDefaultConfig(); defaultConfig.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, True); ASSERT_TRUE(defaultConfig.getProperty(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING).getVisible()); @@ -146,9 +145,9 @@ TEST_F(MqttSubscriberFbTest, PropertyVisibility) TEST_F(MqttSubscriberFbTest, Config) { StartUp(); - auto config = clientMqttFb.getAvailableFunctionBlockTypes().get(SUB_FB_NAME).createDefaultConfig(); - + auto config = MqttSubscriberFbImpl::CreateType().createDefaultConfig(); config.setPropertyValue(PROPERTY_NAME_SUB_TOPIC, buildTopicName()); + daq::FunctionBlockPtr subFb; ASSERT_NO_THROW(subFb = clientMqttFb.addFunctionBlock(SUB_FB_NAME, config)); @@ -168,7 +167,7 @@ TEST_F(MqttSubscriberFbTest, CreationWithDefaultConfig) StartUp(); daq::FunctionBlockPtr subFb; ASSERT_NO_THROW(subFb = clientMqttFb.addFunctionBlock(SUB_FB_NAME)); - EXPECT_EQ(subFb.getSignals().getCount(), 0u); + EXPECT_EQ(subFb.getSignals(daq::search::Any()).getCount(), 0u); ASSERT_EQ(subFb.getStatusContainer().getStatus("ComponentStatus"), Enumeration("ComponentStatusType", "Error", daqInstance.getContext().getTypeManager())); } @@ -181,7 +180,7 @@ TEST_F(MqttSubscriberFbTest, CreationWithPartialConfig) auto config = PropertyObject(); config.addProperty(StringProperty(PROPERTY_NAME_SUB_TOPIC, String(buildTopicName()))); ASSERT_NO_THROW(subFb = clientMqttFb.addFunctionBlock(SUB_FB_NAME, config)); - EXPECT_EQ(subFb.getSignals().getCount(), 0u); + EXPECT_EQ(subFb.getSignals(daq::search::Any()).getCount(), 0u); ASSERT_EQ(subFb.getStatusContainer().getStatus("ComponentStatus"), Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager())); } @@ -194,8 +193,9 @@ TEST_F(MqttSubscriberFbTest, CreationWithCustomConfig) auto config = clientMqttFb.getAvailableFunctionBlockTypes().get(SUB_FB_NAME).createDefaultConfig(); config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, True); config.setPropertyValue(PROPERTY_NAME_SUB_TOPIC, buildTopicName()); + config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE, static_cast(SDSM::SystemTime)); ASSERT_NO_THROW(subFb = clientMqttFb.addFunctionBlock(SUB_FB_NAME, config)); - EXPECT_EQ(subFb.getSignals().getCount(), 1u); + EXPECT_EQ(subFb.getSignals(daq::search::Any()).getCount(), 2u); ASSERT_EQ(subFb.getStatusContainer().getStatus("ComponentStatus"), Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager())); } @@ -208,6 +208,7 @@ TEST_F(MqttSubscriberFbTest, PreviewSignal) config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, True); config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING, False); config.setPropertyValue(PROPERTY_NAME_SUB_TOPIC, buildTopicName()); + config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE, static_cast(SDSM::SystemTime)); ASSERT_NO_THROW(subFb = clientMqttFb.addFunctionBlock(SUB_FB_NAME, config)); ASSERT_EQ(subFb.getSignals().getCount(), 1u); EXPECT_EQ(subFb.getSignals()[0].getDescriptor().getSampleType(), daq::SampleType::Binary); @@ -215,6 +216,25 @@ TEST_F(MqttSubscriberFbTest, PreviewSignal) EXPECT_EQ(subFb.getSignals()[0].getDescriptor().getSampleType(), daq::SampleType::String); } +TEST_F(MqttSubscriberFbTest, DomainForPreviewSignal) +{ + StartUp(); + daq::FunctionBlockPtr subFb; + auto config = clientMqttFb.getAvailableFunctionBlockTypes().get(SUB_FB_NAME).createDefaultConfig(); + config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, True); + config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_IS_STRING, False); + config.setPropertyValue(PROPERTY_NAME_SUB_TOPIC, buildTopicName()); + config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE, static_cast(SDSM::None)); + ASSERT_NO_THROW(subFb = clientMqttFb.addFunctionBlock(SUB_FB_NAME, config)); + ASSERT_EQ(subFb.getSignals(daq::search::Any()).getCount(), 1u); + subFb.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE, static_cast(SDSM::SystemTime)); + ASSERT_EQ(subFb.getSignals(daq::search::Any()).getCount(), 2u); + ASSERT_TRUE(subFb.getSignals()[0].getDomainSignal().assigned()); + subFb.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE, static_cast(SDSM::None)); + ASSERT_EQ(subFb.getSignals(daq::search::Any()).getCount(), 1u); + ASSERT_FALSE(subFb.getSignals()[0].getDomainSignal().assigned()); +} + TEST_F(MqttSubscriberFbTest, SubscriptionStatusWaitingForData) { StartUp(); @@ -463,17 +483,25 @@ TEST_F(MqttSubscriberFbTest, DataTransfer) std::vector{0x31}, std::vector{0x41, 0x42, 0x43, 0x44, 0x45}}; std::vector> dataToReceive; + std::vector tsToReceive; + std::vector timePoints; - CreateSubFB({topic}); + CreateSubFB(topic, true, SDSM::SystemTime); - auto signalList = List(); - obj->getSignals(&signalList); + auto signalList = getSignals(); auto reader = daq::PacketReader(signalList[0]); + auto getTime = []() + { + using namespace std::chrono; + return duration_cast(system_clock::now().time_since_epoch()).count(); + }; + timePoints.push_back(getTime()); for (const auto& data : dataToSend) { mqtt::MqttMessage msg = {topic, data, 1, 0}; onSignalsMessage(msg); + timePoints.push_back(getTime()); } while (!reader.getEmpty()) @@ -488,10 +516,21 @@ TEST_F(MqttSubscriberFbTest, DataTransfer) std::vector readData(dataPacket.getDataSize()); memcpy(readData.data(), dataPacket.getData(), dataPacket.getDataSize()); dataToReceive.push_back(std::move(readData)); + const auto domainPacket = dataPacket.getDomainPacket(); + if (domainPacket.assigned()) + tsToReceive.push_back(*(reinterpret_cast(domainPacket.getRawData()))); } } ASSERT_EQ(dataToSend.size(), dataToReceive.size()); ASSERT_EQ(dataToSend, dataToReceive); + + ASSERT_EQ(dataToSend.size(), tsToReceive.size()); + ASSERT_EQ(timePoints.size(), tsToReceive.size() + 1); + for (size_t i = 0; i < tsToReceive.size(); ++i) + { + EXPECT_GE(tsToReceive[i], timePoints[i]); + EXPECT_LE(tsToReceive[i], timePoints[i + 1]); + } } TEST_F(MqttSubscriberFbTest, CheckRawFbFullDataTransfer) From 81a03d6c72c30c659e94fcafb2f64019e20f7ddb Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Thu, 12 Mar 2026 17:40:22 +0100 Subject: [PATCH 10/15] refactoring --- .../mqtt_subscriber_fb_impl.h | 19 +- .../src/mqtt_json_decoder_fb_impl.cpp | 2 +- .../src/mqtt_publisher_fb_impl.cpp | 5 +- .../src/mqtt_subscriber_fb_impl.cpp | 24 +- .../include/JsonConfigWrapper.h | 33 + .../include/MqttAsyncClient.h | 82 +- .../include/MqttDataWrapper.h | 161 ++-- .../mqtt_streaming_protocol/include/common.h | 45 + .../src/CMakeLists.txt | 3 + .../src/JsonConfigWrapper.cpp | 193 +++++ .../src/MqttAsyncClient.cpp | 46 +- .../src/MqttDataWrapper.cpp | 803 +++++++----------- .../tests/test_mqtt_streaming_protocol.cpp | 2 +- 13 files changed, 736 insertions(+), 682 deletions(-) create mode 100644 shared/mqtt_streaming_protocol/include/JsonConfigWrapper.h create mode 100644 shared/mqtt_streaming_protocol/include/common.h create mode 100644 shared/mqtt_streaming_protocol/src/JsonConfigWrapper.cpp diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h index 409aa6a8..a7bca31e 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h @@ -30,20 +30,6 @@ class MqttSubscriberFbImpl final : public FunctionBlock friend class MqttJsonDecoderFbHelper; public: - struct CmdResult - { - bool success = false; - std::string msg; - int token = 0; - - CmdResult(bool success = false, const std::string& msg = "", int token = 0) - : success(success), - msg(msg), - token(token) - { - } - }; - enum class DomainSignalMode : EnumType { None = 0, @@ -67,7 +53,6 @@ class MqttSubscriberFbImpl final : public FunctionBlock std::shared_ptr subscriber; int qos = DEFAULT_SUB_QOS; - mqtt::MqttDataWrapper jsonDataWorker; std::string topicForSubscribing; DictObjectPtr nestedFbTypes; std::vector nestedFunctionBlocks; @@ -103,8 +88,8 @@ class MqttSubscriberFbImpl final : public FunctionBlock void propertyChanged(); bool setTopic(std::string topic); - CmdResult subscribeToTopic(); - CmdResult unsubscribeFromTopic(); + mqtt::CmdResult subscribeToTopic(); + mqtt::CmdResult unsubscribeFromTopic(); void removed() override; DictPtr onGetAvailableFunctionBlockTypes() override; diff --git a/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp index cb2f7abc..dc635acb 100644 --- a/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp @@ -13,7 +13,7 @@ MqttJsonDecoderFbImpl::MqttJsonDecoderFbImpl(const ContextPtr& ctx, const FunctionBlockTypePtr& type, const PropertyObjectPtr& config) : FunctionBlock(type, ctx, parent, generateLocalId()), - jsonDataWorker(loggerComponent) + jsonDataWorker() { initComponentStatus(); if (config.assigned()) diff --git a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp index 6a7b619a..6d25ff52 100644 --- a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp @@ -6,6 +6,7 @@ #include #include #include +#include "JsonConfigWrapper.h" BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE @@ -18,7 +19,7 @@ MqttPublisherFbImpl::MqttPublisherFbImpl(const ContextPtr& ctx, const PropertyObjectPtr& config) : FunctionBlock(type, ctx, parent, generateLocalId()), mqttClient(mqttClient), - jsonDataWorker(loggerComponent), + jsonDataWorker(), running(true), hasSignalError(false), signalDescriptorChanged(false), @@ -512,7 +513,7 @@ void MqttPublisherFbImpl::readProperties() if (config.mode == PublisherMode::Json && config.topicMode == TopicMode::Single) { - auto result = mqtt::MqttDataWrapper::validateTopic(config.topicName, loggerComponent); + auto result = mqtt::JsonConfigWrapper::validateTopic(config.topicName, loggerComponent); hasSettingError = !result.success; settingErrors.push_back(std::move(result.msg)); } diff --git a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp index 25f6d227..d1808a1c 100644 --- a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp @@ -7,6 +7,7 @@ #include #include #include +#include "JsonConfigWrapper.h" BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE @@ -21,7 +22,6 @@ MqttSubscriberFbImpl::MqttSubscriberFbImpl(const ContextPtr& ctx, const PropertyObjectPtr& config) : FunctionBlock(type, ctx, parent, generateLocalId()), subscriber(subscriber), - jsonDataWorker(loggerComponent), topicForSubscribing(""), nestedFbTypes(nullptr), enablePreview(false), @@ -259,11 +259,11 @@ std::pair MqttSubscriberFbImpl::readFileToString(const std::s void MqttSubscriberFbImpl::setJsonConfig(const std::string config) { - jsonDataWorker.setConfig(config); - auto result = jsonDataWorker.isJsonValid(); + mqtt::JsonConfigWrapper jsonConfigWrapper(config); + auto result = jsonConfigWrapper.isJsonValid(); if (result.success) { - auto topic = jsonDataWorker.extractTopic(); + auto topic = jsonConfigWrapper.extractTopic(); result.success = setTopic(topic); if (result.success) { @@ -273,7 +273,7 @@ void MqttSubscriberFbImpl::setJsonConfig(const std::string config) objPtr.setPropertyValue(PROPERTY_NAME_SUB_TOPIC, String(topic)); event.unmute(); } - if (const auto signalDscs = jsonDataWorker.extractDescription(); !signalDscs.empty()) + if (const auto signalDscs = jsonConfigWrapper.extractDescription(); !signalDscs.empty()) { auto fbConfig = MqttJsonDecoderFbImpl::CreateType().createDefaultConfig(); for (const auto& [signalName, descriptor] : signalDscs) @@ -316,7 +316,7 @@ void MqttSubscriberFbImpl::propertyChanged() bool MqttSubscriberFbImpl::setTopic(std::string topic) { - const auto validationStatus = mqtt::MqttDataWrapper::validateTopic(topic, loggerComponent); + const auto validationStatus = mqtt::JsonConfigWrapper::validateTopic(topic, loggerComponent); if (validationStatus.success) { LOG_I("An MQTT topic: {}", topic); @@ -539,9 +539,9 @@ void MqttSubscriberFbImpl::clearSubscribedTopic() topicForSubscribing.clear(); } -MqttSubscriberFbImpl::CmdResult MqttSubscriberFbImpl::subscribeToTopic() +mqtt::CmdResult MqttSubscriberFbImpl::subscribeToTopic() { - CmdResult result{false}; + mqtt::CmdResult result{false}; if (subscriber) { auto lambda = [this](const mqtt::MqttAsyncClient& client, mqtt::MqttMessage& msg) { this->onSignalsMessage(client, msg); }; @@ -563,7 +563,7 @@ MqttSubscriberFbImpl::CmdResult MqttSubscriberFbImpl::subscribeToTopic() LOG_D("Trying to subscribe to the topic: {}", topic); setComponentStatusWithMessage(ComponentStatus::Ok, "Waiting for data for the topic: " + topicForSubscribing); waitingForData = true; - result = {true, "", result.token}; + result = {true, ""}; } } else @@ -581,16 +581,16 @@ MqttSubscriberFbImpl::CmdResult MqttSubscriberFbImpl::subscribeToTopic() return result; } -MqttSubscriberFbImpl::CmdResult MqttSubscriberFbImpl::unsubscribeFromTopic() +mqtt::CmdResult MqttSubscriberFbImpl::unsubscribeFromTopic() { - CmdResult result{true}; + mqtt::CmdResult result{true}; if (subscriber) { const auto topic = getSubscribedTopic(); if (!topic.empty()) { subscriber->setMessageArrivedCb(topic, nullptr); - mqtt::CmdResult unsubRes = subscriber->unsubscribe(topic); + mqtt::MqttAsyncClient::CmdResultWithToken unsubRes = subscriber->unsubscribe(topic); if (unsubRes.success) unsubRes = subscriber->waitForCompletion(unsubRes.token, MQTT_FB_UNSUBSCRIBE_TOUT); diff --git a/shared/mqtt_streaming_protocol/include/JsonConfigWrapper.h b/shared/mqtt_streaming_protocol/include/JsonConfigWrapper.h new file mode 100644 index 00000000..d2febbb3 --- /dev/null +++ b/shared/mqtt_streaming_protocol/include/JsonConfigWrapper.h @@ -0,0 +1,33 @@ +#pragma once + +#include +#include +#include +#include +#include "common.h" + +namespace mqtt +{ + +class JsonConfigWrapper final +{ +public: + JsonConfigWrapper(const std::string& config); + + static CmdResult validateTopic(const daq::StringPtr topic, const daq::LoggerComponentPtr loggerComponent = nullptr); + + std::vector> extractDescription(); + std::string extractTopic(); + CmdResult isJsonValid(); + +private: + rapidjson::Document doc; + std::string config; + + daq::UnitPtr extractSignalUnit(const rapidjson::Value& signalObj); + std::string extractValueFieldName(const rapidjson::Value& signalObj); + std::string extractTimestampFieldName(const rapidjson::Value& signalObj); + std::string extractFieldName(const rapidjson::Value& signalObj, const std::string& field); + +}; +} // namespace mqtt diff --git a/shared/mqtt_streaming_protocol/include/MqttAsyncClient.h b/shared/mqtt_streaming_protocol/include/MqttAsyncClient.h index c50f5210..f1d3cfa5 100644 --- a/shared/mqtt_streaming_protocol/include/MqttAsyncClient.h +++ b/shared/mqtt_streaming_protocol/include/MqttAsyncClient.h @@ -16,44 +16,44 @@ enum class MqttConnectionStatus { pending, }; -struct CmdResult -{ - bool success = false; - std::string msg; - int token = 0; - - CmdResult() - : success(false), - msg(""), - token(0) - { - } - CmdResult(bool success) - : success(success), - msg(""), - token(0) - { - } - - CmdResult(bool success, const std::string& msg) - : success(success), - msg(msg), - token(0) - { - } - - CmdResult(bool success, const std::string& msg, int token) - : success(success), - msg(msg), - token(token) - { - } -}; - class MqttAsyncClient final { public: using MsgArrivedCb_type = void(const MqttAsyncClient &, mqtt::MqttMessage &msg); + struct CmdResultWithToken + { + bool success = false; + std::string msg; + int token = 0; + + CmdResultWithToken() + : success(false), + msg(""), + token(0) + { + } + CmdResultWithToken(bool success) + : success(success), + msg(""), + token(0) + { + } + + CmdResultWithToken(bool success, const std::string& msg) + : success(success), + msg(msg), + token(0) + { + } + + CmdResultWithToken(bool success, const std::string& msg, int token) + : success(success), + msg(msg), + token(token) + { + } + }; + MqttAsyncClient(); MqttAsyncClient(std::string serverUrl, std::string clientId, bool cleanSession); MqttAsyncClient(const MqttAsyncClient &) = delete; @@ -62,20 +62,20 @@ class MqttAsyncClient final { MqttAsyncClient &operator=(MqttAsyncClient &&) = delete; ~MqttAsyncClient(); - CmdResult connect(); - CmdResult disconnect(); + CmdResultWithToken connect(); + CmdResultWithToken disconnect(); bool syncDisconnect(int timeoutMs); - CmdResult publish(const std::string &topic, + CmdResultWithToken publish(const std::string &topic, void *data, size_t dataLen, int qos = 1, bool retained = false); - CmdResult subscribe(std::string topic, int qos); - CmdResult unsubscribe(std::string topic); - CmdResult unsubscribe(const std::vector& topics); - CmdResult waitForCompletion(int token, unsigned long toutMs); + CmdResultWithToken subscribe(std::string topic, int qos); + CmdResultWithToken unsubscribe(std::string topic); + CmdResultWithToken unsubscribe(const std::vector& topics); + CmdResultWithToken waitForCompletion(int token, unsigned long toutMs); void setConnectedCb(std::function cb); void setMessageArrivedCb(std::string topic, std::function cb); diff --git a/shared/mqtt_streaming_protocol/include/MqttDataWrapper.h b/shared/mqtt_streaming_protocol/include/MqttDataWrapper.h index 7a1dbbea..882dd087 100644 --- a/shared/mqtt_streaming_protocol/include/MqttDataWrapper.h +++ b/shared/mqtt_streaming_protocol/include/MqttDataWrapper.h @@ -3,99 +3,41 @@ #include #include #include +#include #include -#include +#include "common.h" #include #include #include -#include namespace mqtt { -template +template struct container_traits { static constexpr bool is_vector = false; using value_type = T; }; -template +template struct container_traits> { static constexpr bool is_vector = true; using value_type = U; }; -template +template inline constexpr bool is_std_vector_v = container_traits::is_vector; -template +template using sample_type_t = typename container_traits::value_type; -struct SampleData -{ - double value; - uint64_t timestamp; -}; - -struct SignalId -{ - std::string topic; - std::string signalName; - - bool operator==(const SignalId& other) const noexcept - { - return topic == other.topic && signalName == other.signalName; - } -}; - -struct DataPackets -{ - daq::DataPacketPtr dataPacket; - daq::DataPacketPtr domainDataPacket; -}; - -struct MqttMsgDescriptor -{ - std::string valueFieldName; // Value - std::string tsFieldName; // Timestamp - daq::UnitPtr unit; -}; - class MqttDataWrapper final { public: - struct CmdResult - { - bool success = false; - std::string msg; - - CmdResult() - : success(false), - msg("") - { - } - CmdResult(bool success, const std::string& msg = "") - : success(success), - msg(msg) - { - } - - CmdResult addError(const std::string& newmsg) - { - success = false; - msg += newmsg; - return *this; - } - CmdResult merge(const CmdResult& other) - { - success = success && other.success; - msg += other.msg; - return *this; - } - }; + using ValueVariant = std::variant, std::vector, std::vector>; enum class DomainSignalMode : int { @@ -105,70 +47,69 @@ class MqttDataWrapper final _count }; - MqttDataWrapper(daq::LoggerComponentPtr loggerComponent); + MqttDataWrapper(); - static CmdResult validateTopic(const daq::StringPtr topic, const daq::LoggerComponentPtr loggerComponent = nullptr); - - void setConfig(const std::string& config); - std::vector> extractDescription(); - std::string extractTopic(); - CmdResult isJsonValid(); void setOutputSignal(daq::SignalConfigPtr outputSignal); CmdResult createAndSendDataPacket(const std::string& json, const uint64_t externalTs); - //bool hasDomainSignal(const SignalId& signalId) const; void setValueFieldName(std::string valueFieldName); void setTimestampFieldName(std::string tsFieldName); void setDomainSignalMode(DomainSignalMode mode); private: - rapidjson::Document doc; - std::string config; + struct DataPackets + { + daq::DataPacketPtr dataPacket; + daq::DataPacketPtr domainDataPacket; + }; + + struct ExtractionContext + { + rapidjson::Document jsonDocument; + ValueVariant value{}; + std::vector ts; + std::vector outputData; + CmdResult result{true}; + bool tsExtracted = false; + bool valueExtracted = false; + bool jsonParsed = false; + bool dataPacketBuilt = false; + }; - daq::LoggerComponentPtr loggerComponent; daq::SignalConfigPtr outputSignal; // used for description how to extract data from sample json MqttMsgDescriptor msgDescriptor; - DomainSignalMode domainSignalType; + DomainSignalMode domainSignalMode; - std::pair> - extractDataSamples(const std::string& json, const uint64_t externalTs); + std::pair> extractDataSamples(const std::string& json, const uint64_t externalTs); void sendDataSamples(const DataPackets& dataPackets); + template - DataPackets buildDataPackets(const T& value, uint64_t timestamp); + std::vector buildDataPackets(const std::vector& value, const std::vector& timestamp); template - DataPackets buildDataPackets(const T& value, const std::vector& timestamp); + std::vector buildDataPackets(const std::vector& value); template - DataPackets buildDataPackets(const T& value); - daq::DataPacketPtr buildDomainDataPacket(daq::GenericSignalConfigPtr<> signalConfig, uint64_t timestamp); - daq::DataPacketPtr buildDomainDataPacket(daq::GenericSignalConfigPtr<> signalConfig, const std::vector& timestamp); - template - daq::DataPacketPtr buildDataPacket(daq::GenericSignalConfigPtr<> signalConfig, - const T& value, - const daq::DataPacketPtr domainPacket); - template - daq::DataPacketPtr createEmptyDataPacket(const daq::GenericSignalConfigPtr<> signalConfig, - const daq::DataPacketPtr domainPacket, const T& value); - template void copyDataIntoPacket(daq::DataPacketPtr dataPacket, const T& value); - daq::UnitPtr extractSignalUnit(const rapidjson::Value& signalObj); - std::string extractValueFieldName(const rapidjson::Value& signalObj); - std::string extractTimestampFieldName(const rapidjson::Value& signalObj); - std::string extractFieldName(const rapidjson::Value& signalObj, const std::string& field); + DataPackets buildDataPacketsImpl(const T& value, uint64_t timestamp); + template + DataPackets buildDataPacketsImpl(const std::vector& value, const std::vector& timestamp); + + template + daq::DataPacketPtr buildDataPacket(daq::SignalConfigPtr signalConfig, const T& value, const daq::DataPacketPtr domainPacket); + daq::DataPacketPtr buildDomainDataPacket(daq::SignalConfigPtr signalConfig, uint64_t timestamp); + daq::DataPacketPtr buildDomainDataPacket(daq::SignalConfigPtr signalConfig, const std::vector& timestamp); + template + daq::DataPacketPtr + createEmptyDataPacket(const daq::SignalConfigPtr signalConfig, const daq::DataPacketPtr domainPacket, const T& value); + template + void copyDataIntoPacket(daq::DataPacketPtr dataPacket, const T& value); + + bool parseJson(ExtractionContext& ctx, const std::string& json); + bool parseJsonFields(ExtractionContext& ctx); + bool extractValue(ExtractionContext& ctx, const std::string& jsonFieldName); + bool extractTimestamp(ExtractionContext& ctx, const std::string& jsonFieldName); + bool validateExtractionResult(ExtractionContext& ctx); + bool buildPackets(ExtractionContext& ctx, const uint64_t externalTs); template static bool isTypeTheSame(daq::SampleType sampleType); }; } // namespace mqtt - -namespace std -{ - -template <> struct hash -{ - std::size_t operator()(const mqtt::SignalId& id) const noexcept - { - std::size_t h1 = std::hash{}(id.topic); - std::size_t h2 = std::hash{}(id.signalName); - return h1 ^ (h2 << 1); - } -}; -} // namespace std diff --git a/shared/mqtt_streaming_protocol/include/common.h b/shared/mqtt_streaming_protocol/include/common.h new file mode 100644 index 00000000..df2bdde3 --- /dev/null +++ b/shared/mqtt_streaming_protocol/include/common.h @@ -0,0 +1,45 @@ +#pragma once + +#include +#include + +namespace mqtt +{ + +struct MqttMsgDescriptor +{ + std::string valueFieldName; // Value + std::string tsFieldName; // Timestamp + daq::UnitPtr unit; +}; + +struct CmdResult +{ + bool success = false; + std::string msg; + + CmdResult() + : success(false), + msg("") + { + } + CmdResult(bool success, const std::string& msg = "") + : success(success), + msg(msg) + { + } + + CmdResult addError(const std::string& newmsg) + { + success = false; + msg += newmsg; + return *this; + } + CmdResult merge(const CmdResult& other) + { + success = success && other.success; + msg += other.msg; + return *this; + } +}; +} // namespace mqtt::utils diff --git a/shared/mqtt_streaming_protocol/src/CMakeLists.txt b/shared/mqtt_streaming_protocol/src/CMakeLists.txt index 77fbb0c5..7c5384db 100644 --- a/shared/mqtt_streaming_protocol/src/CMakeLists.txt +++ b/shared/mqtt_streaming_protocol/src/CMakeLists.txt @@ -2,13 +2,16 @@ set(LIB_NAME mqtt_streaming_protocol) set(SRC_Cpp MqttAsyncClient.cpp MqttDataWrapper.cpp + JsonConfigWrapper.cpp ) set(SRC_PublicHeaders MqttAsyncClient.h MqttMessage.h MqttSettings.h MqttDataWrapper.h + JsonConfigWrapper.h timestampConverter.h + common.h ) set(INCLUDE_DIR ../include) diff --git a/shared/mqtt_streaming_protocol/src/JsonConfigWrapper.cpp b/shared/mqtt_streaming_protocol/src/JsonConfigWrapper.cpp new file mode 100644 index 00000000..6fb479e7 --- /dev/null +++ b/shared/mqtt_streaming_protocol/src/JsonConfigWrapper.cpp @@ -0,0 +1,193 @@ +#include "JsonConfigWrapper.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace mqtt +{ + +JsonConfigWrapper::JsonConfigWrapper(const std::string& config) + : config(config) +{ + doc.Parse(config.c_str()); +} + +CmdResult JsonConfigWrapper::validateTopic(const daq::StringPtr topic, const daq::LoggerComponentPtr loggerComponent) +{ + + CmdResult result(true, ""); + if (!topic.assigned() || topic.getLength() == 0) + { + result = CmdResult(false, "Empty topic is not allowed!"); + if (loggerComponent.assigned()) + { + LOG_W("{}", result.msg); + } + return result; + } + + std::vector list; + boost::split(list, topic.toStdString(), boost::is_any_of("/")); + + for (const auto& part : list) + { + if (part == "#" || part == "+") + { + result = CmdResult(false, fmt::format("Wildcard characters '+' and '#' are not allowed in topic: {}", topic.toStdString())); + if (loggerComponent.assigned()) + { + LOG_W("{}", result.msg); + } + return result; + } + } + + return result; +} + +std::vector> JsonConfigWrapper::extractDescription() +{ + std::vector> result; + if (config.empty() || !isJsonValid().success) + return result; + + auto it = doc.MemberBegin(); + const rapidjson::Value& array = it->value; + // Each topic array contains one or more signal objects + for (const auto& elem : array.GetArray()) + { + // Iterate over signals inside this element + for (auto sigIt = elem.MemberBegin(); sigIt != elem.MemberEnd(); ++sigIt) + { + std::string SignalName(sigIt->name.GetString()); + const rapidjson::Value& signalObj = sigIt->value; + MqttMsgDescriptor msgDescriptor; + msgDescriptor.unit = extractSignalUnit(signalObj); + msgDescriptor.valueFieldName = extractValueFieldName(signalObj); + msgDescriptor.tsFieldName = extractTimestampFieldName(signalObj); + result.push_back(std::pair(std::move(SignalName), std::move(msgDescriptor))); + } + } + + return result; +} + +std::string JsonConfigWrapper::extractTopic() +{ + std::string topic; + if (config.empty() || !isJsonValid().success) + return topic; + + topic = doc.MemberBegin()->name.GetString(); + return topic; +} + +CmdResult JsonConfigWrapper::isJsonValid() +{ + rapidjson::Document doc; + + if (config.empty()) + return CmdResult(true); + + if (doc.Parse(config.c_str()).HasParseError()) + return CmdResult(false, "The JSON config has an invalid format"); + + if (!doc.IsObject()) + return CmdResult(false, "The JSON config must be an object"); + + if (doc.MemberCount() != 1) + return CmdResult(false, "The JSON config must contain exactly one topic"); + + const auto& arrayValue = doc.MemberBegin()->value; + if (!arrayValue.IsArray()) + return CmdResult(false, "The JSON config has wrong format (expected array of signals)"); + + if (!arrayValue.Empty()) + { + for (const auto& signalEntry : arrayValue.GetArray()) + { + if (!signalEntry.IsObject()) + return CmdResult(false, "Each signal entry must be an object"); + + if (signalEntry.MemberCount() != 1) + return CmdResult(false, "Each signal entry must contain exactly one signal"); + + const auto& signal = *signalEntry.MemberBegin(); + const auto& signalBody = signal.value; + + if (!signalBody.IsObject()) + return CmdResult(false, "Signal definition must be an object"); + + if (!signalBody.HasMember("Value")) + { + return CmdResult(false, "Signal must contain a Value field"); + } + + if (!signalBody["Value"].IsString()) + return CmdResult(false, "Signal field 'Value' must be a string"); + + if (signalBody.HasMember("Timestamp") && !signalBody["Timestamp"].IsString()) + return CmdResult(false, "Signal field 'Timestamp' must be a string"); + + if (signalBody.HasMember("Unit")) + { + const auto& unit = signalBody["Unit"]; + if (!unit.IsArray() || unit.Empty()) + return CmdResult(false, "Signal field 'Unit' must be a non-empty array"); + for (const auto& u : unit.GetArray()) + { + if (!u.IsString()) + return CmdResult(false, "Each Unit entry must be a string"); + } + } + } + } + return CmdResult(true); +} + +daq::UnitPtr JsonConfigWrapper::extractSignalUnit(const rapidjson::Value& signalObj) +{ + daq::UnitPtr unit; + if (signalObj.HasMember("Unit") && signalObj["Unit"].IsArray()) + { + auto unitBuilder = daq::UnitBuilder(); + auto unitArr = signalObj["Unit"].GetArray(); + if (unitArr.Size() >= 1 && unitArr[0].IsString()) + { + unitBuilder.setSymbol(unitArr[0].GetString()); + } + if (unitArr.Size() >= 2 && unitArr[1].IsString()) + { + unitBuilder.setName(unitArr[1].GetString()); + } + if (unitArr.Size() >= 3 && unitArr[2].IsString()) + { + unitBuilder.setQuantity(unitArr[2].GetString()); + } + unit = unitBuilder.build(); + } + return unit; +} + +std::string JsonConfigWrapper::extractValueFieldName(const rapidjson::Value& signalObj) +{ + return extractFieldName(signalObj, "Value"); +} + +std::string JsonConfigWrapper::extractTimestampFieldName(const rapidjson::Value& signalObj) +{ + return extractFieldName(signalObj, "Timestamp"); +} + +std::string JsonConfigWrapper::extractFieldName(const rapidjson::Value& signalObj, const std::string& field) +{ + return (signalObj.HasMember(field) && signalObj[field].IsString()) ? signalObj[field].GetString() : ""; +} +} // namespace mqtt diff --git a/shared/mqtt_streaming_protocol/src/MqttAsyncClient.cpp b/shared/mqtt_streaming_protocol/src/MqttAsyncClient.cpp index 698c62d7..20ec86fc 100644 --- a/shared/mqtt_streaming_protocol/src/MqttAsyncClient.cpp +++ b/shared/mqtt_streaming_protocol/src/MqttAsyncClient.cpp @@ -45,7 +45,7 @@ MqttAsyncClient::~MqttAsyncClient() } } -CmdResult MqttAsyncClient::connect() +MqttAsyncClient::CmdResultWithToken MqttAsyncClient::connect() { if (client != nullptr) { @@ -54,14 +54,14 @@ CmdResult MqttAsyncClient::connect() if (serverUrl.empty() || clientId.empty()) { - return CmdResult(false, "serverUrl or clientId is empty"); + return CmdResultWithToken(false, "serverUrl or clientId is empty"); } int rc = MQTTAsync_createWithOptions(&client, serverUrl.c_str(), clientId.c_str(), MQTTCLIENT_PERSISTENCE_NONE, NULL, &createOpts); if (rc != MQTTASYNC_SUCCESS) { - return CmdResult(false, MQTTAsync_strerror(rc)); + return CmdResultWithToken(false, MQTTAsync_strerror(rc)); } rc = MQTTAsync_setCallbacks(client, @@ -72,33 +72,33 @@ CmdResult MqttAsyncClient::connect() if (rc != MQTTASYNC_SUCCESS) { - return CmdResult(false, MQTTAsync_strerror(rc)); + return CmdResultWithToken(false, MQTTAsync_strerror(rc)); } rc = MQTTAsync_setConnected(client, this, &MqttAsyncClient::onConnected); if (rc != MQTTASYNC_SUCCESS) { - return CmdResult(false, MQTTAsync_strerror(rc)); + return CmdResultWithToken(false, MQTTAsync_strerror(rc)); } rc = MQTTAsync_connect(client, &connOpts); if (rc != MQTTASYNC_SUCCESS) { - return CmdResult(false, MQTTAsync_strerror(rc)); + return CmdResultWithToken(false, MQTTAsync_strerror(rc)); } pendingConnect = true; - return CmdResult(true); + return CmdResultWithToken(true); } -CmdResult MqttAsyncClient::disconnect() +MqttAsyncClient::CmdResultWithToken MqttAsyncClient::disconnect() { if (client == nullptr) - return CmdResult(true, "The client is null"); + return CmdResultWithToken(true, "The client is null"); // It is only the result of the request to disconnect (queuing) auto status = MQTTAsync_disconnect(client, &disconnOpts); bool result = (status == MQTTASYNC_SUCCESS || status == MQTTASYNC_DISCONNECTED); - return CmdResult(result, MQTTAsync_strerror(status)); + return CmdResultWithToken(result, MQTTAsync_strerror(status)); } bool MqttAsyncClient::syncDisconnect(int timeoutMs) @@ -162,22 +162,22 @@ void MqttAsyncClient::setUsernamePasswrod(std::string username, std::string pass connOpts.password = !password.empty() ? password.c_str() : NULL; } -CmdResult MqttAsyncClient::publish(const std::string& topic, void* data, size_t dataLen, int qos, bool retained) +MqttAsyncClient::CmdResultWithToken MqttAsyncClient::publish(const std::string& topic, void* data, size_t dataLen, int qos, bool retained) { std::string tmpErr; if (client == nullptr) { - return CmdResult(false, "MQTTAsync is nullptr"); + return CmdResultWithToken(false, "MQTTAsync is nullptr"); } if (topic.empty()) { - return CmdResult(false, "topic is empty"); + return CmdResultWithToken(false, "topic is empty"); } if (qos > 2 || qos < 0) { - return CmdResult(false, "QoS is wrong"); + return CmdResultWithToken(false, "QoS is wrong"); } MQTTAsync_responseOptions opts = MQTTAsync_responseOptions_initializer; @@ -190,30 +190,30 @@ CmdResult MqttAsyncClient::publish(const std::string& topic, void* data, size_t pubmsg.qos = qos; pubmsg.retained = retained ? 1 : 0; int rc = MQTTAsync_sendMessage(client, topic.c_str(), &pubmsg, &opts); - return CmdResult(rc == MQTTASYNC_SUCCESS, MQTTAsync_strerror(rc), opts.token); + return CmdResultWithToken(rc == MQTTASYNC_SUCCESS, MQTTAsync_strerror(rc), opts.token); } -CmdResult MqttAsyncClient::subscribe(std::string topic, int qos) +MqttAsyncClient::CmdResultWithToken MqttAsyncClient::subscribe(std::string topic, int qos) { MQTTAsync_responseOptions opts = MQTTAsync_responseOptions_initializer; opts.onSuccess = MqttAsyncClient::onSubscribeSuccess; opts.onFailure = MqttAsyncClient::onSubscribeFailure; opts.context = this; int rc = MQTTAsync_subscribe(client, topic.c_str(), qos, &opts); - return CmdResult(rc == MQTTASYNC_SUCCESS, MQTTAsync_strerror(rc), opts.token); + return CmdResultWithToken(rc == MQTTASYNC_SUCCESS, MQTTAsync_strerror(rc), opts.token); } -CmdResult MqttAsyncClient::unsubscribe(std::string topic) +MqttAsyncClient::CmdResultWithToken MqttAsyncClient::unsubscribe(std::string topic) { MQTTAsync_responseOptions opts = MQTTAsync_responseOptions_initializer; opts.onSuccess = (MQTTAsync_onSuccess*)&MqttAsyncClient::onUnsubscribeSuccess; opts.onFailure = (MQTTAsync_onFailure*)&MqttAsyncClient::onUnsubscribeFailure; opts.context = this; int rc = MQTTAsync_unsubscribe(client, topic.c_str(), &opts); - return CmdResult(rc == MQTTASYNC_SUCCESS, MQTTAsync_strerror(rc), opts.token); + return CmdResultWithToken(rc == MQTTASYNC_SUCCESS, MQTTAsync_strerror(rc), opts.token); } -CmdResult MqttAsyncClient::unsubscribe(const std::vector& topics) +MqttAsyncClient::CmdResultWithToken MqttAsyncClient::unsubscribe(const std::vector& topics) { MQTTAsync_responseOptions opts = MQTTAsync_responseOptions_initializer; opts.onSuccess = (MQTTAsync_onSuccess*)&MqttAsyncClient::onUnsubscribeSuccess; @@ -235,15 +235,15 @@ CmdResult MqttAsyncClient::unsubscribe(const std::vector& topics) &opts ); - return CmdResult(rc == MQTTASYNC_SUCCESS, + return CmdResultWithToken(rc == MQTTASYNC_SUCCESS, MQTTAsync_strerror(rc), opts.token); } -CmdResult MqttAsyncClient::waitForCompletion(int token, unsigned long toutMs) +MqttAsyncClient::CmdResultWithToken MqttAsyncClient::waitForCompletion(int token, unsigned long toutMs) { int rc = MQTTAsync_waitForCompletion(client, token, toutMs); - return CmdResult(rc == MQTTASYNC_SUCCESS, MQTTAsync_strerror(rc)); + return CmdResultWithToken(rc == MQTTASYNC_SUCCESS, MQTTAsync_strerror(rc)); } void MqttAsyncClient::setServerURL(std::string serverUrl) diff --git a/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp b/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp index bb718d16..d74bee0d 100644 --- a/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp +++ b/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp @@ -1,25 +1,18 @@ #include "MqttDataWrapper.h" -#include -#include #include -#include #include #include #include #include -#include #include -#include -namespace{ +namespace +{ template -std::vector parseHomogeneousArrayInternal( - const rapidjson::Value::ConstArray& arr, - IsFn isValid, - GetFn getValue) +std::vector parseHomogeneousArrayInternal(const rapidjson::Value::ConstArray& arr, IsFn isValid, GetFn getValue) { std::vector out; out.reserve(arr.Size()); @@ -35,21 +28,17 @@ std::vector parseHomogeneousArrayInternal( } template -std::pair> parseHomogeneousArray( - const rapidjson::Value::ConstArray& arr) +std::pair> parseHomogeneousArray(const rapidjson::Value::ConstArray& arr) { - return std::pair{mqtt::MqttDataWrapper::CmdResult{false, {}}, std::vector{}}; + return std::pair{mqtt::CmdResult{false, {}}, std::vector{}}; } template <> -std::pair> parseHomogeneousArray( - const rapidjson::Value::ConstArray& arr) -{ - std::pair> result{{true, {}}, {}}; - result.second = parseHomogeneousArrayInternal( - arr, - [](const auto& x) { return x.IsInt64() || x.IsUint64(); }, - [](const auto& x) { return x.GetInt64(); }); +std::pair> parseHomogeneousArray(const rapidjson::Value::ConstArray& arr) +{ + std::pair> result{{true, {}}, {}}; + result.second = parseHomogeneousArrayInternal< + int64_t>(arr, [](const auto& x) { return x.IsInt64() || x.IsUint64(); }, [](const auto& x) { return x.GetInt64(); }); if (result.second.empty()) { result.first.addError("Mixed types in value array (expected integers). "); @@ -58,14 +47,11 @@ std::pair> parseHomogeneo } template <> -std::pair> parseHomogeneousArray( - const rapidjson::Value::ConstArray& arr) -{ - std::pair> result{{true, {}}, {}}; - result.second = parseHomogeneousArrayInternal( - arr, - [](const auto& x) { return x.IsUint64(); }, - [](const auto& x) { return x.GetUint64(); }); +std::pair> parseHomogeneousArray(const rapidjson::Value::ConstArray& arr) +{ + std::pair> result{{true, {}}, {}}; + result.second = parseHomogeneousArrayInternal< + uint64_t>(arr, [](const auto& x) { return x.IsUint64(); }, [](const auto& x) { return x.GetUint64(); }); if (result.second.empty()) { result.first.addError("Mixed types in value array (expected unsigned integers). "); @@ -74,14 +60,11 @@ std::pair> parseHomogene } template <> -std::pair> parseHomogeneousArray( - const rapidjson::Value::ConstArray& arr) -{ - std::pair> result{{true, {}}, {}}; - result.second = parseHomogeneousArrayInternal( - arr, - [](const auto& x) { return x.IsDouble(); }, - [](const auto& x) { return x.GetDouble(); }); +std::pair> parseHomogeneousArray(const rapidjson::Value::ConstArray& arr) +{ + std::pair> result{{true, {}}, {}}; + result.second = + parseHomogeneousArrayInternal(arr, [](const auto& x) { return x.IsDouble(); }, [](const auto& x) { return x.GetDouble(); }); if (result.second.empty()) { result.first.addError("Mixed types in value array (expected doubles). "); @@ -90,422 +73,301 @@ std::pair> parseHomogeneou } template <> -std::pair> parseHomogeneousArray( - const rapidjson::Value::ConstArray& arr) -{ - std::pair> result{{true, {}}, {}}; - result.second = parseHomogeneousArrayInternal( - arr, - [](const auto& x) { return x.IsString(); }, - [](const auto& x) { return std::string{x.GetString()}; }); +std::pair> parseHomogeneousArray(const rapidjson::Value::ConstArray& arr) +{ + std::pair> result{{true, {}}, {}}; + result.second = parseHomogeneousArrayInternal< + std::string>(arr, [](const auto& x) { return x.IsString(); }, [](const auto& x) { return std::string{x.GetString()}; }); if (result.second.empty()) { result.first.addError("Mixed types in value array (expected strings). "); } return result; } -} +} // namespace namespace mqtt { -MqttDataWrapper::MqttDataWrapper(daq::LoggerComponentPtr loggerComponent) - : loggerComponent(loggerComponent), - domainSignalType(DomainSignalMode::None) +MqttDataWrapper::MqttDataWrapper() + : domainSignalMode(DomainSignalMode::None) { } -MqttDataWrapper::CmdResult MqttDataWrapper::validateTopic(const daq::StringPtr topic, const daq::LoggerComponentPtr loggerComponent) +void MqttDataWrapper::setOutputSignal(daq::SignalConfigPtr outputSignal) { + this->outputSignal = outputSignal; +} - MqttDataWrapper::CmdResult result(true, ""); - if (!topic.assigned() || topic.getLength() == 0) - { - result = MqttDataWrapper::CmdResult(false, "Empty topic is not allowed!"); - if (loggerComponent.assigned()) - { - LOG_W("{}", result.msg); - } - return result; - } - - std::vector list; - boost::split(list, topic.toStdString(), boost::is_any_of("/")); - - for (const auto& part : list) +CmdResult MqttDataWrapper::createAndSendDataPacket(const std::string& json, const uint64_t externalTs) +{ + auto [status, packets] = extractDataSamples(json, externalTs); + if (status.success) { - if (part == "#" || part == "+") + for (const auto& data : packets) { - result = MqttDataWrapper::CmdResult(false, - fmt::format("Wildcard characters '+' and '#' are not allowed in topic: {}", - topic.toStdString())); - if (loggerComponent.assigned()) - { - LOG_W("{}", result.msg); - } - return result; + sendDataSamples(data); } } + return status; +} - return result; +void MqttDataWrapper::setValueFieldName(std::string valueFieldName) +{ + msgDescriptor.valueFieldName = std::move(valueFieldName); } -void MqttDataWrapper::setConfig(const std::string& config) +void MqttDataWrapper::setTimestampFieldName(std::string tsFieldName) { - this->config = config; - doc.Parse(config.c_str()); + msgDescriptor.tsFieldName = std::move(tsFieldName); } -std::vector> MqttDataWrapper::extractDescription() +void MqttDataWrapper::setDomainSignalMode(DomainSignalMode mode) { - std::vector> result; - if (config.empty() || !isJsonValid().success) - return result; + domainSignalMode = mode; +} - auto it = doc.MemberBegin(); - const rapidjson::Value& array = it->value; - // Each topic array contains one or more signal objects - for (const auto& elem : array.GetArray()) +bool MqttDataWrapper::parseJson(ExtractionContext& ctx, const std::string& json) +{ + ctx.jsonParsed = true; + try { - // Iterate over signals inside this element - for (auto sigIt = elem.MemberBegin(); sigIt != elem.MemberEnd(); ++sigIt) + ctx.jsonDocument.Parse(json); + if (ctx.jsonDocument.HasParseError()) + { + ctx.result.addError("Error parsing MQTT payload as JSON. "); + ctx.jsonParsed = false; + } + else if (!ctx.jsonDocument.IsObject()) { - std::string SignalName(sigIt->name.GetString()); - const rapidjson::Value& signalObj = sigIt->value; - MqttMsgDescriptor msgDescriptor; - msgDescriptor.unit = extractSignalUnit(signalObj); - msgDescriptor.valueFieldName = extractValueFieldName(signalObj); - msgDescriptor.tsFieldName = extractTimestampFieldName(signalObj); - result.push_back(std::pair(std::move(SignalName), std::move(msgDescriptor))); + ctx.result.addError("MQTT payload has wrong format. "); + ctx.jsonParsed = false; } } - - return result; -} - -std::string MqttDataWrapper::extractTopic() -{ - std::string topic; - if (config.empty() || !isJsonValid().success) - return topic; - - topic = doc.MemberBegin()->name.GetString(); - return topic; + catch (...) + { + ctx.result.addError("Error deserializing MQTT payload. "); + ctx.jsonParsed = false; + } + return ctx.jsonParsed; } -MqttDataWrapper::CmdResult MqttDataWrapper::isJsonValid() +bool MqttDataWrapper::parseJsonFields(ExtractionContext& ctx) { - rapidjson::Document doc; - - if (config.empty()) - return CmdResult(true); - - if (doc.Parse(config.c_str()).HasParseError()) - return CmdResult(false, "The JSON config has an invalid format"); - - if (!doc.IsObject()) - return CmdResult(false, "The JSON config must be an object"); - - if (doc.MemberCount() != 1) - return CmdResult(false, "The JSON config must contain exactly one topic"); - - const auto& arrayValue = doc.MemberBegin()->value; - if (!arrayValue.IsArray()) - return CmdResult(false, "The JSON config has wrong format (expected array of signals)"); - - if (!arrayValue.Empty()) + if (!ctx.jsonParsed) + return false; + try { - for (const auto& signalEntry : arrayValue.GetArray()) + for (auto it = ctx.jsonDocument.MemberBegin(); it != ctx.jsonDocument.MemberEnd(); ++it) { - if (!signalEntry.IsObject()) - return CmdResult(false, "Each signal entry must be an object"); - - if (signalEntry.MemberCount() != 1) - return CmdResult(false, "Each signal entry must contain exactly one signal"); - - const auto& signal = *signalEntry.MemberBegin(); - const auto& signalBody = signal.value; - - if (!signalBody.IsObject()) - return CmdResult(false, "Signal definition must be an object"); - - if (!signalBody.HasMember("Value")) + const std::string name = it->name.GetString(); + bool processed = false; + processed |= extractValue(ctx, name); + if (domainSignalMode == DomainSignalMode::ExtractFromMessage && !processed) { - return CmdResult(false, "Signal must contain a Value field"); + processed |= extractTimestamp(ctx, name); } - - if (!signalBody["Value"].IsString()) - return CmdResult(false, "Signal field 'Value' must be a string"); - - if (signalBody.HasMember("Timestamp") && !signalBody["Timestamp"].IsString()) - return CmdResult(false, "Signal field 'Timestamp' must be a string"); - - if (signalBody.HasMember("Unit")) + if (!processed) { - const auto& unit = signalBody["Unit"]; - if (!unit.IsArray() || unit.Empty()) - return CmdResult(false, "Signal field 'Unit' must be a non-empty array"); - for (const auto& u : unit.GetArray()) - { - if (!u.IsString()) - return CmdResult(false, "Each Unit entry must be a string"); - } + // the field is not precessed } } } - return CmdResult(true); -} - -void MqttDataWrapper::setOutputSignal(daq::SignalConfigPtr outputSignal) -{ - this->outputSignal = outputSignal; + catch (...) + { + ctx.result.addError("Error deserializing MQTT payload. "); + return false; + } + return true; } -MqttDataWrapper::CmdResult MqttDataWrapper::createAndSendDataPacket(const std::string& json, const uint64_t externalTs) +bool MqttDataWrapper::extractValue(ExtractionContext& ctx, const std::string& jsonFieldName) { - auto [status, packets] = extractDataSamples(json, externalTs); - if (status.success) + bool fieldFound = (!msgDescriptor.valueFieldName.empty() && jsonFieldName == msgDescriptor.valueFieldName); + if (fieldFound) { - for (const auto& data : packets) + const auto fillContext = [&ctx](mqtt::CmdResult& parsingStatus, auto&& out) { - sendDataSamples(data); + ctx.result.merge(parsingStatus); + ctx.valueExtracted = parsingStatus.success; + if (parsingStatus.success) + ctx.value = std::move(out); + }; + const auto& v = ctx.jsonDocument[jsonFieldName]; + if (v.IsArray()) + { + const auto& arr = v.GetArray(); + if (arr.Empty()) + { + ctx.result.addError("Value field is an array but it is empty. "); + } + else if (arr[0].IsInt64() || arr[0].IsUint64()) + { + auto [parsingStatus, out] = parseHomogeneousArray(arr); + fillContext(parsingStatus, std::move(out)); + } + else if (arr[0].IsDouble()) + { + auto [parsingStatus, out] = parseHomogeneousArray(arr); + fillContext(parsingStatus, std::move(out)); + } + else if (arr[0].IsString()) + { + auto [parsingStatus, out] = parseHomogeneousArray(arr); + fillContext(parsingStatus, std::move(out)); + } + else + { + ctx.result.addError(fmt::format("Unsupported value type for '{}' array. ", jsonFieldName)); + } + } + else + { + ctx.valueExtracted = true; + if (v.IsInt64()) + ctx.value = std::vector{v.GetInt64()}; + else if (v.IsUint64()) + ctx.value = std::vector{static_cast(v.GetUint64())}; + else if (v.IsDouble()) + ctx.value = std::vector{v.GetDouble()}; + else if (v.IsString()) + ctx.value = std::vector{std::string(v.GetString())}; + else + { + ctx.result.addError(fmt::format("Unsupported value type for '{}'. ", jsonFieldName)); + ctx.valueExtracted = false; + } } } - return status; -} - -void MqttDataWrapper::setValueFieldName(std::string valueFieldName) -{ - msgDescriptor.valueFieldName = std::move(valueFieldName); -} - -void MqttDataWrapper::setTimestampFieldName(std::string tsFieldName) -{ - msgDescriptor.tsFieldName = std::move(tsFieldName); -} - -void MqttDataWrapper::setDomainSignalMode(DomainSignalMode mode) -{ - domainSignalType = mode; + return fieldFound; } -std::pair> MqttDataWrapper::extractDataSamples(const std::string& json, - const uint64_t externalTs) +bool MqttDataWrapper::extractTimestamp(ExtractionContext& ctx, const std::string& jsonFieldName) { - using ValueVariant = std::variant, std::vector, std::vector>; - ValueVariant value{}; - std::vector outputData; - std::vector ts; - bool hasTS = false; - bool hasValue = false; - CmdResult result(true); - - try + bool fieldFound = (!msgDescriptor.tsFieldName.empty() && jsonFieldName == msgDescriptor.tsFieldName); + if (fieldFound) { - rapidjson::Document jsonDocument; - jsonDocument.Parse(json); - if (jsonDocument.HasParseError()) + const auto& tsField = ctx.jsonDocument[jsonFieldName]; + if (tsField.IsArray()) { - return std::pair{result.addError("Error parsing mqtt payload as JSON"), outputData}; - } + const auto& arr = tsField.GetArray(); + if (arr.Empty()) + { + ctx.result.addError("Timestamp field is an array but it is empty. "); + } + else if (arr[0].IsInt() || arr[0].IsUint64() || arr[0].IsInt64()) + { + auto [parsingStatus, out] = parseHomogeneousArray(arr); + ctx.result.merge(parsingStatus); + ctx.tsExtracted = parsingStatus.success; + if (parsingStatus.success) + ctx.ts = std::move(out); - if (jsonDocument.IsObject()) - { - for (auto it = jsonDocument.MemberBegin(); it != jsonDocument.MemberEnd(); ++it) + std::for_each(ctx.ts.begin(), ctx.ts.end(), [](auto& val) { val = mqtt::utils::numericToMicroseconds(val); }); + } + else if (arr[0].IsString()) { - const std::string name = it->name.GetString(); - if (!msgDescriptor.valueFieldName.empty() && name == msgDescriptor.valueFieldName) - { - const auto& v = jsonDocument[name]; - if (v.IsArray()) - { - const auto& arr = v.GetArray(); - if (arr.Empty()) - { - result.addError("Value field is an array but it is empty. "); - } - else if (arr[0].IsInt64() || arr[0].IsUint64()) - { - auto [parsingStatus, out] = parseHomogeneousArray(arr); - result.merge(parsingStatus); - hasValue = parsingStatus.success; - if (parsingStatus.success) - value = std::move(out); - } - else if (arr[0].IsDouble()) - { - auto [parsingStatus, out] = parseHomogeneousArray(arr); - result.merge(parsingStatus); - hasValue = parsingStatus.success; - if (parsingStatus.success) - value = std::move(out); - } - else if (arr[0].IsString()) - { - auto [parsingStatus, out] = parseHomogeneousArray(arr); - result.merge(parsingStatus); - hasValue = parsingStatus.success; - if (parsingStatus.success) - value = std::move(out); - } - else - { - result.addError(fmt::format("Unsupported value type for '{}' array. ", name)); - } - } - else - { - hasValue = true; - if (v.IsInt64()) - value = std::vector{v.GetInt64()}; - else if (v.IsUint64()) - value = std::vector{static_cast(v.GetUint64())}; - else if (v.IsDouble()) - value = std::vector{v.GetDouble()}; - else if (v.IsString()) - value = std::vector{std::string(v.GetString())}; - else - { - result.addError(fmt::format("Unsupported value type for '{}'. ", name)); - hasValue = false; - } - } - } - else if (domainSignalType == DomainSignalMode::ExtractFromMessage && !msgDescriptor.tsFieldName.empty() && - name == msgDescriptor.tsFieldName) + auto [parsingStatus, out] = parseHomogeneousArray(arr); + ctx.result.merge(parsingStatus); + ctx.tsExtracted = parsingStatus.success; + if (parsingStatus.success) { - const auto& tsField = jsonDocument[name]; - if (tsField.IsArray()) - { - const auto& arr = tsField.GetArray(); - if (arr.Empty()) - { - result.addError("Timestamp field is an array but it is empty. "); - } - else if (arr[0].IsInt() || arr[0].IsUint64() || arr[0].IsInt64()) - { - auto [parsingStatus, out] = parseHomogeneousArray(arr); - result.merge(parsingStatus); - hasTS = parsingStatus.success; - if (parsingStatus.success) - ts = std::move(out); - - std::for_each(ts.begin(), ts.end(), [](auto& val) { val = mqtt::utils::numericToMicroseconds(val); }); - } - else if (arr[0].IsString()) - { - std::vector stringTs; - auto [parsingStatus, out] = parseHomogeneousArray(arr); - result.merge(parsingStatus); - hasTS = parsingStatus.success; - if (parsingStatus.success) - stringTs = std::move(out); - ts.reserve(stringTs.size()); - std::for_each(stringTs.cbegin(), - stringTs.cend(), - [&ts](const auto& val) { ts.push_back(utils::toUnixTicks(val)); }); - hasTS = true; - } - else - { - result.addError("Timestamp value type is not supported. "); - } - } - else - { - if (tsField.IsInt() || tsField.IsUint64() || tsField.IsInt64()) - { - ts.push_back(mqtt::utils::numericToMicroseconds(tsField.GetUint64())); - hasTS = true; - } - else if (tsField.IsString()) - { - ts.push_back(utils::toUnixTicks(tsField.GetString())); - hasTS = true; - } - else - { - result.addError("Timestamp value type is not supported. "); - } - } + ctx.ts.reserve(out.size()); + std::for_each(out.cbegin(), out.cend(), [&ctx](const auto& val) { ctx.ts.push_back(utils::toUnixTicks(val)); }); } } + else + { + ctx.result.addError("Timestamp value type is not supported. "); + } + } + else + { + ctx.tsExtracted = true; + if (tsField.IsInt() || tsField.IsUint64() || tsField.IsInt64()) + ctx.ts.push_back(mqtt::utils::numericToMicroseconds(tsField.GetUint64())); + else if (tsField.IsString()) + ctx.ts.push_back(utils::toUnixTicks(tsField.GetString())); + else + { + ctx.result.addError("Timestamp value type is not supported. "); + ctx.tsExtracted = false; + } } } - catch (...) + return fieldFound; +} + +bool MqttDataWrapper::validateExtractionResult(ExtractionContext& ctx) +{ + if (!ctx.jsonParsed) + return false; + + bool valueExtractionError = !ctx.valueExtracted; + bool tsExtractionError = + (domainSignalMode == DomainSignalMode::ExtractFromMessage && !msgDescriptor.tsFieldName.empty() && !ctx.tsExtracted); + bool alignmentError = false; + + if (valueExtractionError || tsExtractionError) { - result.addError("Error deserializing MQTT payload. "); + ctx.result.addError("Not all required fields are present in the JSON message. "); + if (valueExtractionError) + ctx.result.addError(fmt::format("Couldn't extract value field (\"{}\") from the JSON message. ", msgDescriptor.valueFieldName)); + if (tsExtractionError) + ctx.result.addError( + fmt::format("Couldn't extract timestamp field (\"{}\") from the JSON message. ", msgDescriptor.tsFieldName)); } - if (!hasValue || (domainSignalType == DomainSignalMode::ExtractFromMessage && !msgDescriptor.tsFieldName.empty() && !hasTS)) + + auto getSize = [](const ValueVariant& v) { return std::visit([](const auto& vec) { return vec.size(); }, v); }; + + if (domainSignalMode == DomainSignalMode::ExtractFromMessage && ctx.tsExtracted && ctx.ts.size() != getSize(ctx.value)) { - result.addError("Not all required fields are present in the JSON message. "); - if (!hasValue) - result.addError(fmt::format("Couldn't extract value field (\"{}\") from the JSON message. ", msgDescriptor.valueFieldName)); - if (domainSignalType == DomainSignalMode::ExtractFromMessage && !msgDescriptor.tsFieldName.empty() && !hasTS) - result.addError(fmt::format("Couldn't extract timestamp field (\"{}\") from the JSON message. ", msgDescriptor.tsFieldName)); + ctx.result.addError("Timestamp and value array sizes do not match. "); + alignmentError = true; } - - if (result.success) + else if (domainSignalMode == DomainSignalMode::ExternalTimestamp && getSize(ctx.value) != 1) { - std::visit( - [&](auto&& values) - { - using T = std::decay_t; - if (domainSignalType == DomainSignalMode::ExtractFromMessage && hasTS && ts.size() != values.size()) - { - result.addError("Timestamp and value array sizes do not match. "); - return; - } - if (domainSignalType == DomainSignalMode::ExternalTimestamp && values.size() > 1) - { - result.addError("External timestamp mode doesn't support arrays of values. "); - return; - } - if constexpr (std::is_same_v>) - { - for (size_t i = 0; i < values.size(); ++i) - { - DataPackets dp; - if (domainSignalType == DomainSignalMode::ExternalTimestamp) - { - dp = buildDataPackets(values[i], externalTs); - } - else if (domainSignalType == DomainSignalMode::ExtractFromMessage) - { - dp = buildDataPackets(values[i], ts[i]); - } - else if (domainSignalType == DomainSignalMode::None) - { - dp = buildDataPackets(values[i]); - } - - if (dp.dataPacket.assigned()) - outputData.push_back(std::move(dp)); - } - } - else - { - DataPackets dp; - if (domainSignalType == DomainSignalMode::ExternalTimestamp) - { - dp = buildDataPackets(values, externalTs); - } - else if (domainSignalType == DomainSignalMode::ExtractFromMessage) - { - dp = buildDataPackets(values, ts); - } - else if (domainSignalType == DomainSignalMode::None) - { - dp = buildDataPackets(values); - } - - if (dp.dataPacket.assigned()) - outputData.push_back(std::move(dp)); - } - }, - value); + ctx.result.addError("External timestamp mode doesn't support arrays of values. "); + alignmentError = true; } - return std::pair{result, outputData}; + + return (valueExtractionError || tsExtractionError || alignmentError); +} + +bool MqttDataWrapper::buildPackets(ExtractionContext& ctx, const uint64_t externalTs) +{ + ctx.dataPacketBuilt = false; + if (!ctx.result.success) + return ctx.dataPacketBuilt; + + std::visit( + [this, &ctx, externalTs](auto&& values) + { + if (domainSignalMode == DomainSignalMode::ExternalTimestamp) + ctx.ts = std::vector{externalTs}; + + if (domainSignalMode != DomainSignalMode::None) + ctx.outputData = buildDataPackets(values, ctx.ts); + else + ctx.outputData = buildDataPackets(values); + + ctx.dataPacketBuilt = (ctx.outputData.size() == values.size()); + }, + ctx.value); + return ctx.dataPacketBuilt; +} + +std::pair> MqttDataWrapper::extractDataSamples(const std::string& json, + const uint64_t externalTs) +{ + ExtractionContext ctx; + + parseJson(ctx, json); + parseJsonFields(ctx); + validateExtractionResult(ctx); + buildPackets(ctx, externalTs); + + return std::pair{std::move(ctx.result), std::move(ctx.outputData)}; } void MqttDataWrapper::sendDataSamples(const DataPackets& dataPackets) @@ -518,20 +380,50 @@ void MqttDataWrapper::sendDataSamples(const DataPackets& dataPackets) } template -DataPackets -MqttDataWrapper::buildDataPackets(const T& value, uint64_t timestamp) +std::vector MqttDataWrapper::buildDataPackets(const std::vector& value, + const std::vector& timestamp) { - DataPackets dataPackets; + std::vector dataPackets; + if constexpr (std::is_same_v, std::string>) + { + dataPackets.reserve(value.size()); + for (size_t i = 0; i < value.size(); ++i) + dataPackets.push_back(buildDataPacketsImpl(value[i], timestamp[i])); + } + else + { + dataPackets.push_back(buildDataPacketsImpl(value, timestamp)); + } - dataPackets.domainDataPacket = buildDomainDataPacket(outputSignal, timestamp); - dataPackets.dataPacket = buildDataPacket(outputSignal, value, dataPackets.domainDataPacket); + return dataPackets; +} + +template +std::vector MqttDataWrapper::buildDataPackets(const std::vector& value) +{ + std::vector dataPackets; + if constexpr (std::is_same_v, std::string>) + { + dataPackets.reserve(value.size()); + for (size_t i = 0; i < value.size(); ++i) + { + DataPackets pkt; + pkt.dataPacket = buildDataPacket(outputSignal, value[i], daq::DataPacketPtr()); + dataPackets.push_back(std::move(pkt)); + } + } + else + { + DataPackets pkt; + pkt.dataPacket = buildDataPacket(outputSignal, value, daq::DataPacketPtr()); + dataPackets.push_back(std::move(pkt)); + } return dataPackets; } template -DataPackets -MqttDataWrapper::buildDataPackets(const T& value, const std::vector& timestamp) +MqttDataWrapper::DataPackets MqttDataWrapper::buildDataPacketsImpl(const T& value, uint64_t timestamp) { DataPackets dataPackets; @@ -542,11 +434,12 @@ MqttDataWrapper::buildDataPackets(const T& value, const std::vector& t } template -DataPackets -MqttDataWrapper::buildDataPackets(const T& value) +MqttDataWrapper::DataPackets MqttDataWrapper::buildDataPacketsImpl(const std::vector& value, const std::vector& timestamp) { DataPackets dataPackets; - dataPackets.dataPacket = buildDataPacket(outputSignal, value, daq::DataPacketPtr()); + + dataPackets.domainDataPacket = buildDomainDataPacket(outputSignal, timestamp); + dataPackets.dataPacket = buildDataPacket(outputSignal, value, dataPackets.domainDataPacket); return dataPackets; } @@ -554,42 +447,42 @@ MqttDataWrapper::buildDataPackets(const T& value) template bool MqttDataWrapper::isTypeTheSame(daq::SampleType sampleType) { - using daq::SampleTypeToType; - using daq::SampleTypeFromType; using daq::SampleType; + using daq::SampleTypeFromType; + using daq::SampleTypeToType; switch (sampleType) { - case daq::SampleType::Float32: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::Float64: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::UInt8: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::Int8: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::Int16: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::UInt16: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::Int32: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::UInt32: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::Int64: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::UInt64: - return std::is_same_v::Type, TReadType>; - case daq::SampleType::Binary: - case daq::SampleType::String: - return std::is_same_v::Type, typename SampleTypeFromType::Type>; - case daq::SampleType::RangeInt64: - case daq::SampleType::ComplexFloat32: - case daq::SampleType::ComplexFloat64: - case daq::SampleType::Struct: - case daq::SampleType::Invalid: - case daq::SampleType::Null: - case daq::SampleType::_count: - break; + case daq::SampleType::Float32: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::Float64: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::UInt8: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::Int8: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::Int16: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::UInt16: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::Int32: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::UInt32: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::Int64: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::UInt64: + return std::is_same_v::Type, TReadType>; + case daq::SampleType::Binary: + case daq::SampleType::String: + return std::is_same_v::Type, typename SampleTypeFromType::Type>; + case daq::SampleType::RangeInt64: + case daq::SampleType::ComplexFloat32: + case daq::SampleType::ComplexFloat64: + case daq::SampleType::Struct: + case daq::SampleType::Invalid: + case daq::SampleType::Null: + case daq::SampleType::_count: + break; } return false; @@ -597,7 +490,7 @@ bool MqttDataWrapper::isTypeTheSame(daq::SampleType sampleType) template daq::DataPacketPtr -MqttDataWrapper::buildDataPacket(daq::GenericSignalConfigPtr<> signalConfig, const T& value, const daq::DataPacketPtr domainPacket) +MqttDataWrapper::buildDataPacket(daq::SignalConfigPtr signalConfig, const T& value, const daq::DataPacketPtr domainPacket) { const auto curType = signalConfig.getDescriptor().getSampleType(); using ActualType = sample_type_t; @@ -613,8 +506,9 @@ MqttDataWrapper::buildDataPacket(daq::GenericSignalConfigPtr<> signalConfig, con return dataPacket; } -template -daq::DataPacketPtr MqttDataWrapper::createEmptyDataPacket(const daq::GenericSignalConfigPtr<> signalConfig, const daq::DataPacketPtr domainPacket, const T& value) +template +daq::DataPacketPtr +MqttDataWrapper::createEmptyDataPacket(const daq::SignalConfigPtr signalConfig, const daq::DataPacketPtr domainPacket, const T& value) { daq::DataPacketPtr dataPacket; uint64_t size = 1; @@ -665,7 +559,7 @@ void MqttDataWrapper::copyDataIntoPacket(daq::DataPacketPtr dataPacket, const T& } } -daq::DataPacketPtr MqttDataWrapper::buildDomainDataPacket(daq::GenericSignalConfigPtr<> signalConfig, uint64_t timestamp) +daq::DataPacketPtr MqttDataWrapper::buildDomainDataPacket(daq::SignalConfigPtr signalConfig, uint64_t timestamp) { daq::DataPacketPtr dataPacket; if (signalConfig.getDomainSignal().assigned()) @@ -677,7 +571,7 @@ daq::DataPacketPtr MqttDataWrapper::buildDomainDataPacket(daq::GenericSignalConf return dataPacket; } -daq::DataPacketPtr MqttDataWrapper::buildDomainDataPacket(daq::GenericSignalConfigPtr<> signalConfig, const std::vector& timestamp) +daq::DataPacketPtr MqttDataWrapper::buildDomainDataPacket(daq::SignalConfigPtr signalConfig, const std::vector& timestamp) { daq::DataPacketPtr dataPacket; if (signalConfig.getDomainSignal().assigned()) @@ -688,45 +582,4 @@ daq::DataPacketPtr MqttDataWrapper::buildDomainDataPacket(daq::GenericSignalConf return dataPacket; } - -daq::UnitPtr MqttDataWrapper::extractSignalUnit(const rapidjson::Value& signalObj) -{ - daq::UnitPtr unit; - if (signalObj.HasMember("Unit") && signalObj["Unit"].IsArray()) - { - auto unitBuilder = daq::UnitBuilder(); - auto unitArr = signalObj["Unit"].GetArray(); - if (unitArr.Size() >= 1 && unitArr[0].IsString()) - { - unitBuilder.setSymbol(unitArr[0].GetString()); - } - if (unitArr.Size() >= 2 && unitArr[1].IsString()) - { - unitBuilder.setName(unitArr[1].GetString()); - } - if (unitArr.Size() >= 3 && unitArr[2].IsString()) - { - unitBuilder.setQuantity(unitArr[2].GetString()); - } - unit = unitBuilder.build(); - } - return unit; -} - -std::string MqttDataWrapper::extractValueFieldName(const rapidjson::Value& signalObj) -{ - return extractFieldName(signalObj, "Value"); -} - -std::string MqttDataWrapper::extractTimestampFieldName(const rapidjson::Value& signalObj) -{ - return extractFieldName(signalObj, "Timestamp"); -} -std::string MqttDataWrapper::extractFieldName(const rapidjson::Value& signalObj, - const std::string& field) -{ - return (signalObj.HasMember(field) && signalObj[field].IsString()) - ? signalObj[field].GetString() - : ""; -} } // namespace mqtt diff --git a/shared/mqtt_streaming_protocol/tests/test_mqtt_streaming_protocol.cpp b/shared/mqtt_streaming_protocol/tests/test_mqtt_streaming_protocol.cpp index a5aa17a3..d816a612 100644 --- a/shared/mqtt_streaming_protocol/tests/test_mqtt_streaming_protocol.cpp +++ b/shared/mqtt_streaming_protocol/tests/test_mqtt_streaming_protocol.cpp @@ -124,7 +124,7 @@ TEST_F(MqttStreamingProtocolTest, PublishingWithoutDataControl) auto ok = connect("127.0.0.1", clientId); ASSERT_TRUE(ok); - CmdResult result; + MqttAsyncClient::CmdResultWithToken result; std::atomic sendDone{false}; std::promise sendPromise; auto sendFuture = sendPromise.get_future(); From 7e3039bf6914ecd8564fae75ce4b66d1ef2f0b35 Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Thu, 12 Mar 2026 18:12:52 +0100 Subject: [PATCH 11/15] new layout --- .../mqtt_client_fb_impl.h | 4 +-- .../mqtt_json_decoder_fb_impl.h | 2 +- .../mqtt_publisher_fb_impl.h | 4 +-- .../mqtt_subscriber_fb_impl.h | 6 ++-- .../src/mqtt_publisher_fb_impl.cpp | 7 ++-- .../src/mqtt_subscriber_fb_impl.cpp | 7 ++-- .../tests/test_mqtt_json_decoder_fb.cpp | 3 +- .../JsonConfigWrapper.h | 2 -- .../MqttAsyncClient.h | 0 .../MqttDataWrapper.h | 0 .../MqttMessage.h | 0 .../MqttSettings.h | 0 .../{ => mqtt_streaming_protocol}/common.h | 0 .../include/mqtt_streaming_protocol/utils.h | 13 +++++++ .../src/CMakeLists.txt | 5 +-- .../src/JsonConfigWrapper.cpp | 35 +------------------ .../src/MqttAsyncClient.cpp | 2 +- .../src/MqttDataWrapper.cpp | 4 +-- .../timestampConverter.h => src/utils.cpp} | 29 +++++++++++++-- .../tests/MqttAsyncClientWrapper.cpp | 2 +- .../tests/MqttAsyncClientWrapper.h | 2 +- .../tests/test_mqtt_streaming_protocol.cpp | 4 +-- 22 files changed, 69 insertions(+), 62 deletions(-) rename shared/mqtt_streaming_protocol/include/{ => mqtt_streaming_protocol}/JsonConfigWrapper.h (87%) rename shared/mqtt_streaming_protocol/include/{ => mqtt_streaming_protocol}/MqttAsyncClient.h (100%) rename shared/mqtt_streaming_protocol/include/{ => mqtt_streaming_protocol}/MqttDataWrapper.h (100%) rename shared/mqtt_streaming_protocol/include/{ => mqtt_streaming_protocol}/MqttMessage.h (100%) rename shared/mqtt_streaming_protocol/include/{ => mqtt_streaming_protocol}/MqttSettings.h (100%) rename shared/mqtt_streaming_protocol/include/{ => mqtt_streaming_protocol}/common.h (100%) create mode 100644 shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/utils.h rename shared/mqtt_streaming_protocol/{include/timestampConverter.h => src/utils.cpp} (72%) diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_client_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_client_fb_impl.h index f287f07e..7d552800 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_client_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_client_fb_impl.h @@ -15,8 +15,8 @@ */ #pragma once -#include "MqttAsyncClient.h" -#include "MqttSettings.h" +#include "mqtt_streaming_protocol/MqttAsyncClient.h" +#include "mqtt_streaming_protocol/MqttSettings.h" #include #include #include diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h index 3dde5f11..653a7862 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h @@ -15,7 +15,7 @@ */ #pragma once -#include "MqttDataWrapper.h" +#include "mqtt_streaming_protocol/MqttDataWrapper.h" #include #include diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_publisher_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_publisher_fb_impl.h index 13bf299b..100687b9 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_publisher_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_publisher_fb_impl.h @@ -15,8 +15,8 @@ */ #pragma once -#include "MqttAsyncClient.h" -#include "MqttDataWrapper.h" +#include "mqtt_streaming_protocol/MqttAsyncClient.h" +#include "mqtt_streaming_protocol/MqttDataWrapper.h" #include "mqtt_streaming_module/handler_base.h" #include "mqtt_streaming_module/status_helper.h" #include diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h index a7bca31e..51d09f8e 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h @@ -15,12 +15,12 @@ */ #pragma once -#include "MqttAsyncClient.h" -#include "MqttDataWrapper.h" +#include "mqtt_streaming_protocol/MqttAsyncClient.h" #include #include #include "mqtt_streaming_module/constants.h" -#include +#include +#include "mqtt_streaming_protocol/common.h" BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE diff --git a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp index 6d25ff52..bd6d6fb9 100644 --- a/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_publisher_fb_impl.cpp @@ -1,12 +1,13 @@ #include "mqtt_streaming_module/constants.h" #include "mqtt_streaming_module/handler_factory.h" +#include "mqtt_streaming_protocol/JsonConfigWrapper.h" +#include "mqtt_streaming_protocol/utils.h" #include #include #include +#include #include #include -#include -#include "JsonConfigWrapper.h" BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE @@ -513,7 +514,7 @@ void MqttPublisherFbImpl::readProperties() if (config.mode == PublisherMode::Json && config.topicMode == TopicMode::Single) { - auto result = mqtt::JsonConfigWrapper::validateTopic(config.topicName, loggerComponent); + auto result = mqtt::utils::validateTopic(config.topicName); hasSettingError = !result.success; settingErrors.push_back(std::move(result.msg)); } diff --git a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp index d1808a1c..0ec12798 100644 --- a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp @@ -1,13 +1,14 @@ #include "mqtt_streaming_module/constants.h" -#include +#include "mqtt_streaming_protocol/JsonConfigWrapper.h" +#include "mqtt_streaming_protocol/utils.h" #include #include #include #include #include +#include #include #include -#include "JsonConfigWrapper.h" BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE @@ -316,7 +317,7 @@ void MqttSubscriberFbImpl::propertyChanged() bool MqttSubscriberFbImpl::setTopic(std::string topic) { - const auto validationStatus = mqtt::JsonConfigWrapper::validateTopic(topic, loggerComponent); + const auto validationStatus = mqtt::utils::validateTopic(topic); if (validationStatus.success) { LOG_I("An MQTT topic: {}", topic); diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp index 2300ba95..d952bdc3 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp @@ -4,7 +4,7 @@ #include "mqtt_streaming_module/mqtt_json_decoder_fb_impl.h" #include "test_daq_test_helper.h" #include "test_data.h" -#include "timestampConverter.h" +#include "mqtt_streaming_protocol/utils.h" #include #include #include @@ -12,6 +12,7 @@ #include #include #include +#include using namespace daq; using namespace daq::modules::mqtt_streaming_module; diff --git a/shared/mqtt_streaming_protocol/include/JsonConfigWrapper.h b/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/JsonConfigWrapper.h similarity index 87% rename from shared/mqtt_streaming_protocol/include/JsonConfigWrapper.h rename to shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/JsonConfigWrapper.h index d2febbb3..9af2d72b 100644 --- a/shared/mqtt_streaming_protocol/include/JsonConfigWrapper.h +++ b/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/JsonConfigWrapper.h @@ -14,8 +14,6 @@ class JsonConfigWrapper final public: JsonConfigWrapper(const std::string& config); - static CmdResult validateTopic(const daq::StringPtr topic, const daq::LoggerComponentPtr loggerComponent = nullptr); - std::vector> extractDescription(); std::string extractTopic(); CmdResult isJsonValid(); diff --git a/shared/mqtt_streaming_protocol/include/MqttAsyncClient.h b/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/MqttAsyncClient.h similarity index 100% rename from shared/mqtt_streaming_protocol/include/MqttAsyncClient.h rename to shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/MqttAsyncClient.h diff --git a/shared/mqtt_streaming_protocol/include/MqttDataWrapper.h b/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/MqttDataWrapper.h similarity index 100% rename from shared/mqtt_streaming_protocol/include/MqttDataWrapper.h rename to shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/MqttDataWrapper.h diff --git a/shared/mqtt_streaming_protocol/include/MqttMessage.h b/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/MqttMessage.h similarity index 100% rename from shared/mqtt_streaming_protocol/include/MqttMessage.h rename to shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/MqttMessage.h diff --git a/shared/mqtt_streaming_protocol/include/MqttSettings.h b/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/MqttSettings.h similarity index 100% rename from shared/mqtt_streaming_protocol/include/MqttSettings.h rename to shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/MqttSettings.h diff --git a/shared/mqtt_streaming_protocol/include/common.h b/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/common.h similarity index 100% rename from shared/mqtt_streaming_protocol/include/common.h rename to shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/common.h diff --git a/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/utils.h b/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/utils.h new file mode 100644 index 00000000..c2ebf58f --- /dev/null +++ b/shared/mqtt_streaming_protocol/include/mqtt_streaming_protocol/utils.h @@ -0,0 +1,13 @@ +#pragma once + +#include "mqtt_streaming_protocol/common.h" +#include + +namespace mqtt::utils +{ + +mqtt::CmdResult validateTopic(const daq::StringPtr topic); +uint64_t numericToMicroseconds(uint64_t val); +uint64_t toUnixTicks(const std::string& input); + +} // namespace mqtt::utils diff --git a/shared/mqtt_streaming_protocol/src/CMakeLists.txt b/shared/mqtt_streaming_protocol/src/CMakeLists.txt index 7c5384db..918730df 100644 --- a/shared/mqtt_streaming_protocol/src/CMakeLists.txt +++ b/shared/mqtt_streaming_protocol/src/CMakeLists.txt @@ -3,6 +3,7 @@ set(LIB_NAME mqtt_streaming_protocol) set(SRC_Cpp MqttAsyncClient.cpp MqttDataWrapper.cpp JsonConfigWrapper.cpp + utils.cpp ) set(SRC_PublicHeaders MqttAsyncClient.h @@ -10,11 +11,11 @@ set(SRC_PublicHeaders MqttAsyncClient.h MqttSettings.h MqttDataWrapper.h JsonConfigWrapper.h - timestampConverter.h + utils.h common.h ) -set(INCLUDE_DIR ../include) +set(INCLUDE_DIR ../include/${TARGET_FOLDER_NAME}) opendaq_prepend_include(${INCLUDE_DIR} SRC_PublicHeaders) source_group("include" FILES ${SRC_PublicHeaders}) diff --git a/shared/mqtt_streaming_protocol/src/JsonConfigWrapper.cpp b/shared/mqtt_streaming_protocol/src/JsonConfigWrapper.cpp index 6fb479e7..a99ec4c1 100644 --- a/shared/mqtt_streaming_protocol/src/JsonConfigWrapper.cpp +++ b/shared/mqtt_streaming_protocol/src/JsonConfigWrapper.cpp @@ -1,4 +1,4 @@ -#include "JsonConfigWrapper.h" +#include "mqtt_streaming_protocol/JsonConfigWrapper.h" #include #include @@ -19,39 +19,6 @@ JsonConfigWrapper::JsonConfigWrapper(const std::string& config) doc.Parse(config.c_str()); } -CmdResult JsonConfigWrapper::validateTopic(const daq::StringPtr topic, const daq::LoggerComponentPtr loggerComponent) -{ - - CmdResult result(true, ""); - if (!topic.assigned() || topic.getLength() == 0) - { - result = CmdResult(false, "Empty topic is not allowed!"); - if (loggerComponent.assigned()) - { - LOG_W("{}", result.msg); - } - return result; - } - - std::vector list; - boost::split(list, topic.toStdString(), boost::is_any_of("/")); - - for (const auto& part : list) - { - if (part == "#" || part == "+") - { - result = CmdResult(false, fmt::format("Wildcard characters '+' and '#' are not allowed in topic: {}", topic.toStdString())); - if (loggerComponent.assigned()) - { - LOG_W("{}", result.msg); - } - return result; - } - } - - return result; -} - std::vector> JsonConfigWrapper::extractDescription() { std::vector> result; diff --git a/shared/mqtt_streaming_protocol/src/MqttAsyncClient.cpp b/shared/mqtt_streaming_protocol/src/MqttAsyncClient.cpp index 20ec86fc..70486c85 100644 --- a/shared/mqtt_streaming_protocol/src/MqttAsyncClient.cpp +++ b/shared/mqtt_streaming_protocol/src/MqttAsyncClient.cpp @@ -1,4 +1,4 @@ -#include "MqttAsyncClient.h" +#include "mqtt_streaming_protocol/MqttAsyncClient.h" #include namespace mqtt diff --git a/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp b/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp index d74bee0d..43a5ccfd 100644 --- a/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp +++ b/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp @@ -1,4 +1,4 @@ -#include "MqttDataWrapper.h" +#include "mqtt_streaming_protocol/MqttDataWrapper.h" #include #include @@ -6,7 +6,7 @@ #include #include -#include +#include namespace { diff --git a/shared/mqtt_streaming_protocol/include/timestampConverter.h b/shared/mqtt_streaming_protocol/src/utils.cpp similarity index 72% rename from shared/mqtt_streaming_protocol/include/timestampConverter.h rename to shared/mqtt_streaming_protocol/src/utils.cpp index 5533e9db..e52dd8ef 100644 --- a/shared/mqtt_streaming_protocol/include/timestampConverter.h +++ b/shared/mqtt_streaming_protocol/src/utils.cpp @@ -1,7 +1,9 @@ #pragma once +#include "mqtt_streaming_protocol/utils.h" #include #include +#include #include #include #include @@ -9,7 +11,30 @@ namespace mqtt::utils { -inline uint64_t numericToMicroseconds(uint64_t val) +mqtt::CmdResult validateTopic(const daq::StringPtr topic) +{ + CmdResult result(true, ""); + if (!topic.assigned() || topic.getLength() == 0) + { + result = CmdResult(false, "Empty topic is not allowed!"); + return result; + } + + std::vector list; + boost::split(list, topic.toStdString(), boost::is_any_of("/")); + + for (const auto& part : list) + { + if (part == "#" || part == "+") + { + result = CmdResult(false, fmt::format("Wildcard characters '+' and '#' are not allowed in topic: {}", topic.toStdString())); + return result; + } + } + + return result; +} +uint64_t numericToMicroseconds(uint64_t val) { // Determine number of digits int digits = (val == 0) ? 1 : static_cast(std::log10(val)) + 1; @@ -29,7 +54,7 @@ inline uint64_t numericToMicroseconds(uint64_t val) } } -inline uint64_t toUnixTicks(const std::string& input) +uint64_t toUnixTicks(const std::string& input) { std::string str = input; // Trim leading/trailing spaces diff --git a/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.cpp b/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.cpp index 8a80b317..2775407a 100644 --- a/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.cpp +++ b/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.cpp @@ -1,5 +1,5 @@ #include "MqttAsyncClientWrapper.h" -#include "MqttAsyncClient.h" +#include "mqtt_streaming_protocol/MqttAsyncClient.h" #include "mqtt_streaming_helper/timer.h" #include diff --git a/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.h b/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.h index 85579f7c..66b6586c 100644 --- a/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.h +++ b/shared/mqtt_streaming_protocol/tests/MqttAsyncClientWrapper.h @@ -1,5 +1,5 @@ #pragma once -#include "MqttAsyncClient.h" +#include "mqtt_streaming_protocol/MqttAsyncClient.h" #include #include diff --git a/shared/mqtt_streaming_protocol/tests/test_mqtt_streaming_protocol.cpp b/shared/mqtt_streaming_protocol/tests/test_mqtt_streaming_protocol.cpp index d816a612..672feb09 100644 --- a/shared/mqtt_streaming_protocol/tests/test_mqtt_streaming_protocol.cpp +++ b/shared/mqtt_streaming_protocol/tests/test_mqtt_streaming_protocol.cpp @@ -1,7 +1,7 @@ -#include "MqttAsyncClient.h" +#include "mqtt_streaming_protocol/MqttAsyncClient.h" #include "MqttAsyncClientWrapper.h" #include "mqtt_streaming_helper/timer.h" -#include "timestampConverter.h" +#include "mqtt_streaming_protocol/utils.h" #include #include #include From ccde7dc5f1b2a5cba619fd97ec9b2c49f97f4393 Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Fri, 13 Mar 2026 13:26:11 +0100 Subject: [PATCH 12/15] domain value duplication error; tests --- .../mqtt_json_decoder_fb_impl.h | 5 +- .../mqtt_subscriber_fb_impl.h | 2 +- .../src/mqtt_json_decoder_fb_impl.cpp | 27 +++++++-- .../src/mqtt_subscriber_fb_impl.cpp | 6 +- .../tests/test_mqtt_json_decoder_fb.cpp | 51 ++++++++++++++++ .../tests/test_mqtt_subscriber_fb.cpp | 58 +++++++++++++++++++ 6 files changed, 142 insertions(+), 7 deletions(-) diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h index 653a7862..9f322111 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_json_decoder_fb_impl.h @@ -39,7 +39,7 @@ class MqttJsonDecoderFbImpl final : public FunctionBlock const PropertyObjectPtr& config = nullptr); DAQ_MQTT_STREAM_MODULE_API static FunctionBlockTypePtr CreateType(); - void processMessage(const std::string& json, const uint64_t externalTs); + DAQ_MQTT_STREAM_MODULE_API void processMessage(const std::string& json, const uint64_t externalTs); protected: struct FbConfig { @@ -58,7 +58,9 @@ class MqttJsonDecoderFbImpl final : public FunctionBlock std::atomic configValid; std::string configMsg; std::atomic parsingSucceeded; + std::atomic externalTsDuplicate; std::string parsingMsg; + uint64_t lastExternalTs; FbConfig config; @@ -73,6 +75,7 @@ class MqttJsonDecoderFbImpl final : public FunctionBlock void propertyChanged(); void updateStatuses(); + void checkExternalTs(const uint64_t externalTs); }; END_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE diff --git a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h index 51d09f8e..c13c81ee 100644 --- a/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h +++ b/modules/mqtt_streaming_module/include/mqtt_streaming_module/mqtt_subscriber_fb_impl.h @@ -77,7 +77,7 @@ class MqttSubscriberFbImpl final : public FunctionBlock void reconfigureSignal(); void clearSubscribedTopic(); - DataPacketPtr createDomainDataPacket(const uint64_t epochTime); + DAQ_MQTT_STREAM_MODULE_API DataPacketPtr createDomainDataPacket(const uint64_t epochTime); void processMessage(const mqtt::MqttMessage& msg); void initProperties(const PropertyObjectPtr& config); diff --git a/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp index dc635acb..87991252 100644 --- a/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_json_decoder_fb_impl.cpp @@ -13,7 +13,8 @@ MqttJsonDecoderFbImpl::MqttJsonDecoderFbImpl(const ContextPtr& ctx, const FunctionBlockTypePtr& type, const PropertyObjectPtr& config) : FunctionBlock(type, ctx, parent, generateLocalId()), - jsonDataWorker() + jsonDataWorker(), + lastExternalTs(0) { initComponentStatus(); if (config.assigned()) @@ -147,6 +148,7 @@ void MqttJsonDecoderFbImpl::readProperties() jsonDataWorker.setTimestampFieldName(config.tsFieldName); jsonDataWorker.setDomainSignalMode(config.tsMode); waitingData = configValid.load(); + externalTsDuplicate = false; updateStatuses(); } @@ -168,13 +170,29 @@ void MqttJsonDecoderFbImpl::updateStatuses() { setComponentStatusWithMessage(ComponentStatus::Ok, "Waiting for data"); } - else if (parsingSucceeded) + else if (parsingSucceeded == false) { - setComponentStatusWithMessage(ComponentStatus::Ok, "Parsing succeeded"); + setComponentStatusWithMessage(ComponentStatus::Error, "Parsing failed: " + parsingMsg); + } + else if (externalTsDuplicate) + { + setComponentStatusWithMessage(ComponentStatus::Warning, + "Domain signal value for one of the received messages is the same as previous. " + "Data may be lost!"); } else { - setComponentStatusWithMessage(ComponentStatus::Error, "Parsing failed: " + parsingMsg); + setComponentStatusWithMessage(ComponentStatus::Ok, "Parsing succeeded"); + } +} + +void MqttJsonDecoderFbImpl::checkExternalTs(const uint64_t externalTs) +{ + if (config.tsMode == mqtt::MqttDataWrapper::DomainSignalMode::ExternalTimestamp) + { + if (externalTs == lastExternalTs) + externalTsDuplicate = true; + lastExternalTs = externalTs; } } @@ -184,6 +202,7 @@ void MqttJsonDecoderFbImpl::processMessage(const std::string& json, const uint64 { auto lock = this->getRecursiveConfigLock(); waitingData = false; + checkExternalTs(externalTs); auto status = jsonDataWorker.createAndSendDataPacket(json, externalTs); parsingSucceeded = status.success; if (status.success) diff --git a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp index 0ec12798..f22015d0 100644 --- a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp @@ -431,7 +431,11 @@ DataPacketPtr MqttSubscriberFbImpl::createDomainDataPacket(const uint64_t epochT if (lastTsValue == epochTime) { if (statusContainer.getStatus("ComponentStatus") != ComponentStatus::Error) - setComponentStatusWithMessage(ComponentStatus::Warning, "Domain signal value is the same as previous. Data may be lost"); + { + setComponentStatusWithMessage(ComponentStatus::Warning, + "Domain signal value for one of the received messages is the same as previous. " + "Data may be lost!"); + } } lastTsValue = epochTime; } diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp index d952bdc3..3e97f072 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_json_decoder_fb.cpp @@ -1127,3 +1127,54 @@ TEST_F(MqttJsonDecoderFbTest, RemovingNestedFunctionBlock) ASSERT_EQ(subMqttFb.getStatusContainer().getStatus("ComponentStatus"), Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager())); } + +TEST_F(MqttJsonDecoderFbTest, PacketWithTheSameTS) +{ + using namespace std::chrono; + const std::string warnMsg("Domain signal value for one of the received messages is the same as previous."); + StartUp(); + const std::string topic = buildTopicName(); + const auto msgTemplate = VALID_JSON_DATA_1; + const std::string valueF = extractFieldName(msgTemplate, ""); + AddSubFb(topic); + AddDecoderFb(valueF, DDSM::ExternalTimestamp, ""); + + auto fb = reinterpret_cast(*decoderObj); + auto buildMsg = [](int64_t value) + { + return replacePlaceholder(VALID_JSON_DATA_1, "", value); + }; + auto getTime = []() { return duration_cast(system_clock::now().time_since_epoch()).count(); }; + auto getComponentStatus = [&]() { return decoderObj.getStatusContainer().getStatus("ComponentStatus"); }; + auto getStatusMsg = [&]() { return decoderObj.getStatusContainer().getStatusMessage("ComponentStatus").toStdString(); }; + + const auto warning = Enumeration("ComponentStatusType", "Warning", daqInstance.getContext().getTypeManager()); + const auto ok = Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager()); + + const auto ts = getTime(); + + fb->processMessage(buildMsg(1), ts); + + ASSERT_EQ(getComponentStatus(), ok); + + fb->processMessage(buildMsg(2), ts); + EXPECT_EQ(getComponentStatus(), warning); + EXPECT_NE(getStatusMsg().find(warnMsg), std::string::npos); + + fb->processMessage(buildMsg(3), getTime()); + EXPECT_EQ(getComponentStatus(), warning); + EXPECT_NE(getStatusMsg().find(warnMsg), std::string::npos); + + // reconfiguring should reset warning + decoderObj.setPropertyValue(PROPERTY_NAME_DEC_UNIT, "ppm"); + EXPECT_EQ(getComponentStatus(), ok); + EXPECT_EQ(getStatusMsg().find(warnMsg), std::string::npos); + + fb->processMessage(buildMsg(3), getTime()); + EXPECT_EQ(getComponentStatus(), ok); + EXPECT_EQ(getStatusMsg().find(warnMsg), std::string::npos); + + fb->processMessage(buildMsg(3), getTime()); + EXPECT_EQ(getComponentStatus(), ok); + EXPECT_EQ(getStatusMsg().find(warnMsg), std::string::npos); +} diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp index cdd0107d..218f4ead 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_subscriber_fb.cpp @@ -61,6 +61,12 @@ class MqttSubscriberFbHelper mqtt::MqttAsyncClient unused; obj->onSignalsMessage(unused, msg); } + + DataPacketPtr createDomainDataPacket(daq::FunctionBlockPtr subFb, const uint64_t epochTime) + { + auto fb = reinterpret_cast(*subFb); + return fb->createDomainDataPacket(epochTime); + } }; class MqttSubscriberFbTest : public testing::Test, public DaqTestHelper, public MqttSubscriberFbHelper @@ -658,3 +664,55 @@ TEST_F(MqttSubscriberFbTest, CheckRawFbFullDataTransferWithReconfiguring) ASSERT_EQ(dataToSend[1], dataToReceive[0]); } +TEST_F(MqttSubscriberFbTest, DomainDataPacketWithTheSameTS) +{ + using namespace std::chrono; + const std::string warnMsg("Domain signal value for one of the received messages is the same as previous."); + const auto topic = buildTopicName(); + + StartUp(); + + auto config = clientMqttFb.getAvailableFunctionBlockTypes().get(SUB_FB_NAME).createDefaultConfig(); + config.setPropertyValue(PROPERTY_NAME_SUB_TOPIC, topic); + config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL, True); + config.setPropertyValue(PROPERTY_NAME_SUB_PREVIEW_SIGNAL_TS_MODE, static_cast(SDSM::SystemTime)); + auto fb = clientMqttFb.addFunctionBlock(SUB_FB_NAME, config); + + auto getTime = []() { return duration_cast(system_clock::now().time_since_epoch()).count(); }; + auto getComponentStatus = [&]() { return fb.getStatusContainer().getStatus("ComponentStatus"); }; + auto getStatusMsg = [&]() { return fb.getStatusContainer().getStatusMessage("ComponentStatus").toStdString(); }; + + const auto warning = Enumeration("ComponentStatusType", "Warning", daqInstance.getContext().getTypeManager()); + const auto ok = Enumeration("ComponentStatusType", "Ok", daqInstance.getContext().getTypeManager()); + + const auto ts = getTime(); + auto packet = createDomainDataPacket(fb, ts); + + ASSERT_EQ(getComponentStatus(), ok); + + packet = createDomainDataPacket(fb, ts); + + EXPECT_EQ(getComponentStatus(), warning); + EXPECT_NE(getStatusMsg().find(warnMsg), std::string::npos); + + packet = createDomainDataPacket(fb, getTime()); + + EXPECT_EQ(getComponentStatus(), warning); + EXPECT_NE(getStatusMsg().find(warnMsg), std::string::npos); + + // reconfiguring should reset warning + fb.setPropertyValue(PROPERTY_NAME_SUB_QOS, 2); + + EXPECT_EQ(getComponentStatus(), ok); + EXPECT_EQ(getStatusMsg().find(warnMsg), std::string::npos); + + packet = createDomainDataPacket(fb, getTime()); + + EXPECT_EQ(getComponentStatus(), ok); + EXPECT_EQ(getStatusMsg().find(warnMsg), std::string::npos); + + packet = createDomainDataPacket(fb, getTime()); + + EXPECT_EQ(getComponentStatus(), ok); + EXPECT_EQ(getStatusMsg().find(warnMsg), std::string::npos); +} From b18bfde618cc6bda8d59a2fb8b79ce0d684d9fcd Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Fri, 13 Mar 2026 17:30:28 +0100 Subject: [PATCH 13/15] fix --- .../src/atomic_signal_sample_arr_handler.cpp | 2 +- modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp | 1 + .../mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp | 4 ---- shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp | 1 + shared/mqtt_streaming_protocol/src/utils.cpp | 2 -- 5 files changed, 3 insertions(+), 7 deletions(-) diff --git a/modules/mqtt_streaming_module/src/atomic_signal_sample_arr_handler.cpp b/modules/mqtt_streaming_module/src/atomic_signal_sample_arr_handler.cpp index 0f24117f..015de7a1 100644 --- a/modules/mqtt_streaming_module/src/atomic_signal_sample_arr_handler.cpp +++ b/modules/mqtt_streaming_module/src/atomic_signal_sample_arr_handler.cpp @@ -157,7 +157,7 @@ std::string AtomicSignalSampleArrayHandler::toString(const std::string valueFiel MqttDataSamplePtr AtomicSignalSampleArrayHandler::processDataPackets(SignalContext& signalContext) { if (signalBuffers[signalContext.inputPort.getSignal().getGlobalId().toStdString()].data.empty()) - std::make_shared>(); + return std::make_shared>(); const auto signal = signalContext.inputPort.getSignal(); std::string valueFieldName = buildValueFieldName(signalNamesMode, signal); auto msg = toString(valueFieldName, signalContext); diff --git a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp index f22015d0..3c4be773 100644 --- a/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp +++ b/modules/mqtt_streaming_module/src/mqtt_subscriber_fb_impl.cpp @@ -9,6 +9,7 @@ #include #include #include +#include BEGIN_NAMESPACE_OPENDAQ_MQTT_STREAMING_MODULE diff --git a/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp b/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp index 960fbce6..028d5bc2 100644 --- a/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp +++ b/modules/mqtt_streaming_module/tests/test_mqtt_publisher_fb.cpp @@ -1469,8 +1469,6 @@ TEST_P(MqttPublisherFbPTest, TransferRaw) TEST_F(MqttPublisherFbTest, TransferRawString) { - constexpr size_t sampleCnt = 15; - SignalHelper help{}; StartUp(); ASSERT_NO_THROW(CreateRawPublisherFB()); @@ -1537,8 +1535,6 @@ TEST_F(MqttPublisherFbTest, TransferRawString) TEST_F(MqttPublisherFbTest, TransferRawBinaryData) { - constexpr size_t sampleCnt = 15; - SignalHelper help{}; StartUp(); ASSERT_NO_THROW(CreateRawPublisherFB()); diff --git a/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp b/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp index 43a5ccfd..5dfc11a4 100644 --- a/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp +++ b/shared/mqtt_streaming_protocol/src/MqttDataWrapper.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include diff --git a/shared/mqtt_streaming_protocol/src/utils.cpp b/shared/mqtt_streaming_protocol/src/utils.cpp index e52dd8ef..10d4629d 100644 --- a/shared/mqtt_streaming_protocol/src/utils.cpp +++ b/shared/mqtt_streaming_protocol/src/utils.cpp @@ -1,5 +1,3 @@ -#pragma once - #include "mqtt_streaming_protocol/utils.h" #include #include From 36efeaa8df9e13da7d9236acc8a4875492a6b19a Mon Sep 17 00:00:00 2001 From: Viacheslau Date: Fri, 13 Mar 2026 17:49:08 +0100 Subject: [PATCH 14/15] README.md updating --- README.md | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ebaec0ef..96f92e35 100644 --- a/README.md +++ b/README.md @@ -16,41 +16,45 @@ MQTT module for the [OpenDAQ SDK](https://github.com/openDAQ/openDAQ). The modul - **Where**: *mqtt_streaming_module/src/mqtt_client_fb_impl.cpp, include/mqtt_streaming_module/...* - **Purpose**: Represents the MQTT broker as an openDAQ function block - the connection point through which function blocks are created. - **Main properties:** - - *BrokerAddress* (string) - MQTT broker address. It can be an IP address or a hostname. By default, it is set to *"127.0.0.1"*. - - *BrokerPort* (integer) - Port number for the MQTT broker connection. By default, it is set to *1883*. + - *BrokerAddress* (string) — MQTT broker address. It can be an IP address or a hostname. By default, it is set to *"127.0.0.1"*. + - *BrokerPort* (integer) — Port number for the MQTT broker connection. By default, it is set to *1883*. - *Username* (string) — Username for MQTT broker authentication. By default, it is empty. - *Password* (string) — Password for MQTT broker authentication. By default, it is empty. - *ConnectionTimeout* (integer) — Timeout in milliseconds for the initial connection to the MQTT broker. If the connection fails, an exception is thrown. By default, it is set to *3000 ms*. 2) **MQTT publisher Function Block (MQTTJSONPublisherFB)**: - **Where**: *include/mqtt_streaming_module/mqtt_publisher_fb_impl.h, src/mqtt_publisher_fb_impl.cpp* - - **Purpose**: Publishes openDAQ signal data to MQTT topics. There are **four** general data publishing schemes: - 1) One MQTT message per signal / one message per sample / one topic per signal / one timestamp for each sample. Example: *{"AI0": 1.1, "timestamp": 1763716736100000}* + - **Purpose**: Publishes openDAQ signal data to MQTT topics. + There are **five** general data publishing schemes: + 1) Raw data publishing. When using this approach, the data is transmitted in raw binary form without any additional wrapper. Domain data is not transmitted. + 2) JSON message data publishing. One MQTT message per signal / one message per sample / one topic per signal / one timestamp for each sample. Example: *{"AI0": 1.1, "timestamp": 1763716736100000}* - 2) One MQTT message per signal / one message containing several samples / one topic per signal / one timestamp per sample (array of samples). Example: *{"AI0": [1.1, 2.2, 3.3], "timestamps": [1763716736100000, 1763716736200000, 1763716736300000]}* + 3) JSON message data publishing. One MQTT message per signal / one message containing several samples / one topic per signal / one timestamp per sample (array of samples). Example: *{"AI0": [1.1, 2.2, 3.3], "timestamps": [1763716736100000, 1763716736200000, 1763716736300000]}* - 3) One MQTT message for all signals / one message per sample containing all signals / one topic for all signals / one shared timestamp for all signals. Example: *{"AI0": 1.1, "AI1": 2, "timestamp": 1763716736100000}* + 4) JSON message data publishing. One MQTT message for all signals / one message per sample containing all signals / one topic for all signals / one shared timestamp for all signals. Example: *{"AI0": 1.1, "AI1": 2, "timestamp": 1763716736100000}* - 4) One MQTT message for all signals / one message containing several samples for all signals / one topic for all signals / one shared timestamp for all signals (array of samples). Example: *{"AI0": [1.1, 2.2, 3.3], "AI1": [4.1, 4.2, 4.3], "timestamp": [1763716736100000, 1763716736200000, 1763716736300000]}* + 5) JSON message data publishing. One MQTT message for all signals / one message containing several samples for all signals / one topic for all signals / one shared timestamp for all signals (array of samples). Example: *{"AI0": [1.1, 2.2, 3.3], "AI1": [4.1, 4.2, 4.3], "timestamp": [1763716736100000, 1763716736200000, 1763716736300000]}* The schemes are configured through combinations of properties. - **Main properties**: - - *TopicMode* (list) — Selects whether to publish all signals to separate MQTT topics (one per signal, *TopicPerSignal mode*) or to a single topic (*SingleTopic mode*), one for all signals. Choose *0* for *TopicPerSignal* mode and *1* for *SingleTopic* mode. By default, it is set to *TopicPerSignal* mode. + - *Mode* (list) — Selects the mode of publishing (JSON, Raw). Choose *0* for *JSON* mode and *1* for *Raw* mode. In JSON mode, the function block converts signal samples into JSON messages and publishes them to MQTT topics. In Raw mode, the function block publishes raw signal samples to MQTT topics without any conversion. By default it is set to JSON mode. + - *TopicMode* (list) — The property is used **only** in *JSON* mode. Selects whether to publish all signals to separate MQTT topics (one per signal, *TopicPerSignal mode*) or to a single topic (*SingleTopic mode*), one for all signals. Choose *0* for *TopicPerSignal* mode and *1* for *SingleTopic* mode. By default, it is set to *TopicPerSignal* mode. - *QoS* (list) — MQTT Quality of Service level. It can be *0* (at most once), *1* (at least once), or *2* (exactly once). By default, it is set to *1*. - - *Topic* (string) — Topic name for publishing in *SingleTopic* mode. If left empty, the Publisher's *Global ID* is used as the topic name. + - *Topic* (string) — The property is used **only** in *JSON* mode. Topic name for publishing in *SingleTopic* mode. If left empty, the Publisher's *Global ID* is used as the topic name. - *Topics* (list of strings, read-only) — Contains a list of topics used for publishing data in the *TopicPerSignal* mode. The order in the list is the same as the input ports order. - - *GroupValues* (bool) — Enables the use of a sample pack for a signal. By default, it is set to *false*. - - *SignalValueJSONKey* (list) — Describes how to name a JSON value field. By default it is set to *GlobalID*. - - *SamplesPerMessage* (integer) — Sets the size of the sample pack when publishing grouped values. By default, it is set to *1*. + - *GroupValues* (bool) — The property is used **only** in *JSON* mode. Enables the use of a sample pack for a signal. By default, it is set to *false*. + - *SignalValueJSONKey* (list) — The property is used **only** in *JSON* mode. Describes how to name a JSON value field. By default it is set to *GlobalID*. + - *SamplesPerMessage* (integer) — The property is used **only** in *JSON* mode. Sets the size of the sample pack when publishing grouped values. By default, it is set to *1*. - *ReaderWaitPeriod* (integer) — Polling period in milliseconds, specifying how often the server calls an internal reader to collect and publish the connected signals’ data to an MQTT broker. By default, it is set to *20 ms*. - *EnablePreviewSignal* (bool) — Enable the creation of preview signals: one signal in *SingleTopic* mode and one signal per connected input port in *TopicPerSignal* mode. These signals contain the same JSON string data that is published to MQTT topics. - *Schema* (string, read-only) - Describes the general representation of a JSON data packet according to the current function block settings. To configure the publishing schemes, set the properties as follows: - 1) *TopicMode(0), GroupValues(false)*; - 2) *TopicMode(0), GroupValues(true), SamplesPerMessage()*; - 3) *TopicMode(1), GroupValues(false)*; - 4) *TopicMode(1), GroupValues(true), SamplesPerMessage()*; + 1) *Mode(1)*; + 2) *Mode(0), TopicMode(0), GroupValues(false)*; + 3) *Mode(0), TopicMode(0), GroupValues(true), SamplesPerMessage()*; + 4) *Mode(0), TopicMode(1), GroupValues(false)*; + 5) *Mode(0), TopicMode(1), GroupValues(true), SamplesPerMessage()*; 3) **MQTT subscriber Function Block (MQTTSubscriberFB)**: @@ -61,6 +65,7 @@ MQTT module for the [OpenDAQ SDK](https://github.com/openDAQ/openDAQ). The modul - *Topic* (string) — MQTT topic to subscribe to for receiving raw binary data. - *QoS* (list) — MQTT Quality of Service level. It can be *0* (at most once), *1* (at least once), or *2* (exactly once). By default, it is set to *1*. - *EnablePreviewSignal* (bool) — Enable the creation of a preview signal. This signal contains the raw binary data received from an MQTT topic. + - *DomainMode* (list) — Defines the domain of the preview signal. By default it is set to *None* (0), which means that the preview signal doesn't have a timestamp. If set to *System time* (1), the preview signal's timestamp is set to the system time when the MQTT message is received. - *MessageIsString* (bool) — Interpret a received message as a string. - *JSONConfigFile* (string) — path to file with **JSON configuration string**. See the *JSONConfig* property for more details. This property could be set only at creation. It is not visible. - *JSONConfig* (string) — **JSON configuration string** that defines the MQTT topic and the corresponding signals to subscribe to. This property could be set only at creation. It is not visible. A typical string structure: @@ -126,6 +131,7 @@ MQTT module for the [OpenDAQ SDK](https://github.com/openDAQ/openDAQ). The modul - **Purpose**: To parse JSON string data to extract a value and a timestamp, and to send data and domain samples based on this data. - **Main properties**: - *ValueKey* (string) — Specifies the JSON field name from which value data will be extracted. This property is required. It should be contained in the incoming JSON messages. Otherwise, a parsing error will occur. + - *DomainMode* (list) — Defines how the timestamp of the decoded signal is generated. By default it is set to *None* (0), which means that the decoded signal doesn't have a timestamp. If set to *Extract from message* (1), the JSON decoder will try to extract the timestamp from the incoming JSON messages (see *DomainKey* property). If set to *System time* (2), the timestamp of the decoded signal is set to the system time when the JSON message is received. - *DomainKey* (string) — Specifies the JSON field name from which timestamp will be extracted. This property is optional. If it is set it should be contained in the incoming JSON messages. Otherwise, a parsing error will occur. - *Unit* (string) — Specifies the unit symbol for the decoded value. This property is optional. --- From c1caeb61cf1902f90fb08960eaedb35b9dd2e21d Mon Sep 17 00:00:00 2001 From: Aliaksandr Adziareika <8034372+alexadereyko@users.noreply.github.com> Date: Mon, 30 Mar 2026 11:41:53 +0200 Subject: [PATCH 15/15] Migrate to opendaq-cmake-utils v1.0.1 (#14) --- cmake/CMakeLists.txt | 2 +- external/CMakeLists.txt | 4 ++++ opendaq_ref | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 5717a523..90e75510 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -8,7 +8,7 @@ include(FetchContent) FetchContent_Declare( opendaq-cmake-utils GIT_REPOSITORY https://github.com/openDAQ/opendaq-cmake-utils.git - GIT_TAG v1.0.0 + GIT_TAG v1.0.1 GIT_PROGRESS ON ) FetchContent_MakeAvailable(opendaq-cmake-utils) diff --git a/external/CMakeLists.txt b/external/CMakeLists.txt index 6ea9ec01..1a9ccb92 100644 --- a/external/CMakeLists.txt +++ b/external/CMakeLists.txt @@ -1,3 +1,5 @@ +opendaq_get_cmake_mode(_MQTT_CMAKE_MODERN_MODE_SAVED) +opendaq_set_cmake_mode(ANCIENT) opendaq_set_cmake_folder_context(TARGET_FOLDER_NAME) if (${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) @@ -11,3 +13,5 @@ endif() add_subdirectory(boost) add_subdirectory(paho_mqtt_c) add_subdirectory(rapidjson) + +opendaq_set_cmake_mode(${_MQTT_CMAKE_MODERN_MODE_SAVED}) diff --git a/opendaq_ref b/opendaq_ref index bf45b759..d0a69106 100644 --- a/opendaq_ref +++ b/opendaq_ref @@ -1 +1 @@ -49c7618711a47890d3f9b09518c3ffc83b65f157 +4757349a1db30721bead0cb695ec3a147913aef4