fix: (138) solving threading with sqlite

worker/Makefile.main_riscv

@@ -1,9 +1,15 @@
 CC = riscv64-linux-gnu-gcc
 
 DPDK_PREFIX = ./dpdk-riscv-install
+SQLITE_PREFIX = ./sqlite3-riscv-install
 PKG_CONFIG = env PKG_CONFIG_LIBDIR=$(DPDK_PREFIX)/lib/pkgconfig pkg-config
 
-CFLAGS_BASE = -Iinclude -O2 $(shell $(PKG_CONFIG) --cflags libdpdk)
+CFLAGS_BASE = -Iinclude/dpdk_filter -O2 $(shell $(PKG_CONFIG) --cflags libdpdk)
+
+DEBUG ?= 0
+ifeq ($(DEBUG),1)
+    CFLAGS_BASE += -DDEBUG
+endif
 
 LDFLAGS = -L$(DPDK_PREFIX)/lib \
           -Wl,--start-group \
@@ -13,10 +19,11 @@ LDFLAGS = -L$(DPDK_PREFIX)/lib \
           -lrte_net \
           -lrte_log -ldl \
           -lrte_hash \
-          -sqlite3 \
+          -lrte_timer \
           -Wl,--end-group \
-          -latomic
-
+          -latomic  \
+          -L$(SQLITE_PREFIX)/lib \
+          -lsqlite3
 
 SRCS = src/dpdk_filter/main.c src/dpdk_filter/net_port.c src/dpdk_filter/filtr_packets.c src/dpdk_filter/pars_packets.c src/dpdk_filter/proc_packets.c src/dpdk_filter/dns_cache.c
 

worker/Makefile.main_x86

@@ -1,6 +1,11 @@
 CC = gcc
-CFLAGS_BASE = -Iinclude -O2 -msse4.2 -mpclmul -maes
-LDFLAGS = -lrte_eal -lrte_ethdev -lrte_mempool -lrte_mbuf -lrte_bus_vdev -lpthread -lnuma -ldl -lrte_net -lrte_hash -lsqlite3
+CFLAGS_BASE = -Iinclude/dpdk_filter -O2 -msse4.2 -mpclmul -maes
+LDFLAGS = -lrte_eal -lrte_ethdev -lrte_mempool -lrte_mbuf -lrte_bus_vdev -lpthread -lnuma -ldl -lrte_net -lrte_hash -lsqlite3 -lrte_timer
+
+DEBUG ?= 0
+ifeq ($(DEBUG),1)
+    CFLAGS_BASE += -DDEBUG
+endif
 
 SRCS = src/dpdk_filter/main.c src/dpdk_filter/net_port.c src/dpdk_filter/filtr_packets.c src/dpdk_filter/pars_packets.c src/dpdk_filter/proc_packets.c src/dpdk_filter/dns_cache.c
 

worker/README(DPDK FILTRING).md

@@ -0,0 +1,55 @@
+# installing_dpdk.md
+Setting up the environment for cross-compilation and installing dpdk is described in "installing_dpdk.md" on the wiki.
+
+
+
+# Create a pair of veth and TAP device
+
+```bash
+sudo ./scripts/set_virt_dev_for_test_xdp.sh
+```
+The script creates a pair veth0 - veth1
+
+```bash
+sudo ./scripts/set_tap_dev.sh
+```
+The script creates a TAP device tap0
+
+
+# Project assembly
+For real ports (eth0/eth1 + tap0):
+```bash
+make -f Makefile.main_riscv all
+```
+
+For virtual ports (veth0/veth1 + tap0):
+```bash
+make -f Makefile.main_riscv virt
+```
+Defining the -DVIRT_PORTS macro switches the program to use virtual interfaces.
+
+Before starting, it is recommended to run the virtual device configuration script:
+```bash
+sudo ./scripts/set_virt_dev_for_test_xdp.sh
+```
+
+For debugging add DEBUG=1, example:
+```bash
+make -f Makefile.main_x86 virt DEBUG=1
+```
+
+# Clean
+```bash
+make -f Makefile.main_riscv clean
+```
+
+# Launch
+The program requires superuser rights (to work with DPDK and XDP):
+```bash
+sudo ./main-riscv-virt
+```
+
+
+# Notes
+The DNS cache is automatically saved to cache.db (SQLite) and restored on restart.
+Periodic saving of the cache occurs every hour using DPDK timers.

worker/helper for association with Worker.md

@@ -1,26 +1,33 @@
-REQUESTED_CLASSIFICATION структура для передачи от контроллера к воркеру:
+REQUESTED_CLASSIFICATION - структура для передачи от контроллера к воркеру:
+
+```code
 struct requested_classification {
-    char get_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN] - политика
-    int get_trust_level                              - уровень доверия к сайту
+    char get_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN]
+    int get_trust_level
 }
+```
 
+Структура для хранения категории с минимальным уровнем доверия для этой категории:
 
-Структура для хранения категории с минимальным уровнем доверия для этой категории  
+```code
 struct trust_categories_with_lvl {
     char locked_by_trust_category[CATEGORY_MAX_LEN];
     int trust_lvl;
 }
+```
 
+у нас есть переменные, которые получаем при инициализации воркера и заносим в структуру (периодически обновляем):
 
-у нас есть переменные, которые получаем при инициализации воркера и заносим в структуру (периодически обновляем)
+```code
 struct BASE_POLICY {
     char locked_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN];
     struct trust_categories_with_lvl categories_with_lvl[MAX_CATEGORIES_BY_TRUST_LVL];
     char block_domains[MAX_DOMAINS][MAX_LEN_DOMEIN];
     char allow_domains[MAX_DOMAINS][MAX_LEN_DOMEIN];
     int min_trust_level;
 }
+```
 
 
+Добавлен tap порт, по которому проходят пакеты исключений в ядро, обрабатываются и ответ отсылается на входящий порт (port_in)
 
-Добавлен tap порт, по которому проходят пакеты исключений в ядро, обрабатываются и ответ отсылается на входящий порт (port_in) 

worker/include/dpdk_filter/constants.h

@@ -1,6 +1,7 @@
 #ifndef CONSTANTS_H
 #define CONSTANTS_H
 
+#include <stdint.h>
 
 #define MAX_CATEGORIES_BY_TRUST_LVL 64
 #define MAX_DOMAINS 64
@@ -10,6 +11,6 @@
 #define CATEGORY_MAX_LEN 64
 #define DNS_CACHE_DEFAULT_TTL (7 * 24 * 60 * 60)
 #define LEN_LIST_EXCEPTION_PORTS 1
-extern const uint16_t  LIST_EXCEPTION_PORTS[LEN_LIST_EXCEPTION_PORTS];
+extern const uint16_t LIST_EXCEPTION_PORTS[LEN_LIST_EXCEPTION_PORTS];
 
 #endif

worker/include/dpdk_filter/dns_cache.h

@@ -6,15 +6,20 @@
 #include <rte_hash.h>
 #include <rte_jhash.h>
 #include <rte_malloc.h>
+#include <rte_timer.h>
+#include <sqlite3.h>
 #include <stdbool.h>
 #include <stdint.h>
-#include <sqlite3.h>
-
-#include "../../include/dpdk_filter/constants.h"
-#include "../../include/dpdk_filter/types.h"
-
 
+#include "constants.h"
+#include "types.h"
 
+void load_cache_from_sqlite(void);
+void close_sqlite_cache(void);
+int save_single_node_to_sqlite(const char *domain, struct node_cache *node);
+void *save_all_cache_to_sqlite(void *arg);
+void init_tables_sqlite_dns_cache(void);
+void copy_data_from_hash_to_snapshot(struct snapshot *snapt);
 
 void init_dns_cache(void);
 int lookup_dns_cache(const char *domain, struct node_cache **return_node);

worker/include/dpdk_filter/filtr_packets.h

@@ -1,22 +1,29 @@
 #ifndef FILTR_PAK_H
 #define FILTR_PAK_H
 
+#include "constants.h"
 #include "pars_packets.h"
+#include "types.h"
 #include <rte_mbuf.h>
 #include <stdint.h>
-#include "../../include/dpdk_filter/constants.h"
-#include "../../include/dpdk_filter/types.h"
 
-bool check_is_block(char domain[DOMAIN_MAX_LEN], char block_domains[MAX_DOMAINS][DOMAIN_MAX_LEN]);
+bool check_is_block(char domain[DOMAIN_MAX_LEN],
+                    char block_domains[MAX_DOMAINS][DOMAIN_MAX_LEN]);
 
-bool check_is_allow(char domain[DOMAIN_MAX_LEN], char allow_domains[MAX_DOMAINS][DOMAIN_MAX_LEN]);
+bool check_is_allow(char domain[DOMAIN_MAX_LEN],
+                    char allow_domains[MAX_DOMAINS][DOMAIN_MAX_LEN]);
 
 bool check_trust_level(int get_trust_level, int min_trust_level);
 
-bool check_categories(char get_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN], char locked_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN]);
+bool check_categories(char get_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN],
+                      char locked_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN]);
 
-bool check_categories_with_lvl(struct requested_classification* req_clas, struct trust_categories_with_lvl categories_with_lvl[MAX_CATEGORIES_BY_TRUST_LVL]);
+bool check_categories_with_lvl(
+    struct requested_classification *req_clas,
+    struct trust_categories_with_lvl
+        categories_with_lvl[MAX_CATEGORIES_BY_TRUST_LVL]);
 
-bool main_filtring(struct requested_classification* req_clas, struct BASE_POLICY* policy, char domain[DOMAIN_MAX_LEN]);
+bool main_filtring(struct requested_classification *req_clas,
+                   struct BASE_POLICY *policy, char domain[DOMAIN_MAX_LEN]);
 
 #endif

worker/include/dpdk_filter/net_port.h

@@ -1,18 +1,15 @@
 #ifndef AF_XDP_PORT_H
 #define AF_XDP_PORT_H
 
+#include "types.h"
 #include <rte_mempool.h>
 #include <stdint.h>
-#include "../../include/dpdk_filter/types.h"
-
-
 
 struct net_port *init_struct_tap_port(const char *tap_iface_name,
-                                            struct rte_mempool *mbuf_pool);
-
+                                      struct rte_mempool *mbuf_pool);
 
 struct net_port *init_struct_af_xdp_port(const char *iface_name,
-                                            struct rte_mempool *mbuf_pool);
+                                         struct rte_mempool *mbuf_pool);
 
 int net_port_init(struct net_port *port);
 

worker/include/dpdk_filter/pars_packets.h

@@ -1,12 +1,10 @@
 #ifndef PARS_PAK_H
 #define PARS_PAK_H
 
+#include "constants.h"
+#include "types.h"
 #include <rte_mbuf.h>
 #include <stdint.h>
-#include "../../include/dpdk_filter/constants.h"
-#include "../../include/dpdk_filter/types.h"
-
-
 
 void parsing_pakage(struct rte_mbuf *paket, struct info_of_pakage *info_pac);
 

worker/include/dpdk_filter/proc_packets.h

@@ -1,24 +1,20 @@
 #ifndef PROC_PAK_H
 #define PROC_PAK_H
 
-#include "../../include/dpdk_filter/net_port.h"
-#include "../../include/dpdk_filter/filtr_packets.h"
-#include "../../include/dpdk_filter/pars_packets.h"
-#include "../../include/dpdk_filter/constants.h"
-#include "../../include/dpdk_filter/types.h"
+#include "constants.h"
+#include "filtr_packets.h"
+#include "net_port.h"
+#include "pars_packets.h"
+#include "types.h"
 #include <rte_ethdev.h>
 #include <rte_mbuf.h>
 #include <rte_mempool.h>
 #include <stdint.h>
 #include <stdlib.h>
 
-
-
-
-
-
-void pakage_processing(struct net_port *port_in,
-                       struct net_port *port_out, struct net_port *port_exception, uint16_t queue_number,
-                       uint16_t nb_pkts, struct rte_mbuf **pkts, struct BASE_POLICY* policy);
+void pakage_processing(struct net_port *port_in, struct net_port *port_out,
+                       struct net_port *port_exception, uint16_t queue_number,
+                       uint16_t nb_pkts, struct rte_mbuf **pkts,
+                       struct BASE_POLICY *policy);
 
 #endif

worker/include/dpdk_filter/types.h

@@ -2,8 +2,27 @@
 #define TYPES_H
 
 #include "constants.h"
-#include <stdint.h>
 #include <stdbool.h>
+#include <stdint.h>
+
+#ifdef DEBUG
+#define LOG_INFO(info, ...)                                                    \
+  fprintf(stdout, "[INFO] %s: %d: " info "\n", __func__, __LINE__,             \
+          ##__VA_ARGS__)
+
+#define LOG_ERROR(error, ...)                                                  \
+  fprintf(stdout, "[ERROR] %s: %d: " error "\n", __func__, __LINE__,           \
+          ##__VA_ARGS__)
+
+#else
+#define LOG_INFO(info, ...)                                                    \
+  do {                                                                         \
+  } while (0)
+
+#define LOG_ERROR(error, ...)                                                  \
+  fprintf(stdout, "[ERROR] %s: %d: " error "\n", __func__, __LINE__,           \
+          ##__VA_ARGS__)
+#endif
 
 struct net_port {
   uint16_t port_id;
@@ -20,24 +39,24 @@ struct info_of_pakage {
 };
 
 struct trust_categories_with_lvl {
-    char locked_by_trust_category[CATEGORY_MAX_LEN];
-    int trust_lvl;
+  char locked_by_trust_category[CATEGORY_MAX_LEN];
+  int trust_lvl;
 };
 
 struct BASE_POLICY {
   char locked_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN];
-  struct trust_categories_with_lvl categories_with_lvl[MAX_CATEGORIES_BY_TRUST_LVL];
+  struct trust_categories_with_lvl
+      categories_with_lvl[MAX_CATEGORIES_BY_TRUST_LVL];
   char block_domains[MAX_DOMAINS][DOMAIN_MAX_LEN];
   char allow_domains[MAX_DOMAINS][DOMAIN_MAX_LEN];
   int min_trust_level;
 };
 
 struct requested_classification {
-    char get_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN];
-    int get_trust_level;
+  char get_categories[MAX_CATEGORIES][CATEGORY_MAX_LEN];
+  int get_trust_level;
 };
 
-
 struct node_cache {
   char categories[MAX_CATEGORIES][CATEGORY_MAX_LEN];
   bool solution_is_send;
@@ -47,5 +66,9 @@ struct node_cache {
   char *key_domain;
 };
 
+struct snapshot {
+  struct node_cache node;
+  char domain[DOMAIN_MAX_LEN];
+};
 
 #endif

worker/scripts/set_tap_dev.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+TAP="tap0"
+
+sudo ip tuntap add $TAP mode tap
+sudo ip link set $TAP up
+sudo ip addr add 10.0.3.1/24 dev $TAP