From c42b2677f16ea357a5093d4099584d11750baf2a Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Fri, 24 Oct 2025 11:42:34 -0400 Subject: [PATCH 01/18] fix(sonicwall): optimize field parsing and clean double quotes --- filters/sonicwall/sonic_wall.conf | 60 ++++++++++++++++++++++++++++++- 1 file changed, 59 insertions(+), 1 deletion(-) diff --git a/filters/sonicwall/sonic_wall.conf b/filters/sonicwall/sonic_wall.conf index 2214f0beb..aa3467496 100644 --- a/filters/sonicwall/sonic_wall.conf +++ b/filters/sonicwall/sonic_wall.conf @@ -57,6 +57,12 @@ filter { } } } + #Remove unnecessary spaces from the msg_all field + if ([msg_all]){ + mutate { + strip => ["msg_all"] + } + } #Checking if the log is in CEF format if ("CEF:" in [message] and [message] =~/\|(\w+)?(\s)?SonicWall(\s)?(\w+)?\|/){ #......................................................................# @@ -222,6 +228,58 @@ filter { grok { match => { "msg_all" => [" cs3=%{DATA:vpnpolicyDst} %{WORD:}="," cs3=%{GREEDYDATA:vpnpolicyDst}"] } } } + + # Clean double quotes from extracted fields + mutate { + gsub => [ + "fw_action", "^\"", "", + "fw_action", "\"$", "", + "msg", "^\"", "", + "msg", "\"$", "", + "note", "^\"", "", + "note", "\"$", "", + "rule", "^\"", "", + "rule", "\"$", "", + "user", "^\"", "", + "user", "\"$", "", + "vpnpolicy", "^\"", "", + "vpnpolicy", "\"$", "", + "vpnpolicyDst", "^\"", "", + "vpnpolicyDst", "\"$", "", + "fileid", "^\"", "", + "fileid", "\"$", "", + "referer", "^\"", "", + "referer", "\"$", "", + "time", "^\"", "", + "time", "\"$", "", + "cfs_category", "^\"", "", + "cfs_category", "\"$", "", + "ipscat", "^\"", "", + "ipscat", "\"$", "", + "app_cat", "^\"", "", + "app_cat", "\"$", "", + "appid", "^\"", "", + "appid", "\"$", "", + "arg", "^\"", "", + "arg", "\"$", "", + "app_name", "^\"", "", + "app_name", "\"$", "", + "contentObject", "^\"", "", + "contentObject", "\"$", "", + "request", "^\"", "", + "request", "\"$", "", + "anti_spyware_cat", "^\"", "", + "anti_spyware_cat", "\"$", "", + "anti_spyware_pri", "^\"", "", + "anti_spyware_pri", "\"$", "", + "mgmt_source_ip", "^\"", "", + "mgmt_source_ip", "\"$", "", + "radio", "^\"", "", + "radio", "\"$", "", + "station", "^\"", "", + "station", "\"$", "" + ] + } #Checking if the src field exists and using grok to parse it if ([src]){ grok { @@ -292,7 +350,7 @@ filter { rename => { "[af_policy_id]" => "[logx][sonicwall][af_policy_id]" } rename => { "[af_policy]" => "[logx][sonicwall][af_policy]" } rename => { "[af_policy_type]" => "[logx][sonicwall][af_policy_type]" } - rename => { "[=%{WORD:af_policy_service]" => "[logx][sonicwall][af_policy_service]" } + rename => { "[af_policy_service]" => "[logx][sonicwall][af_policy_service]" } rename => { "[af_policy_action]" => "[logx][sonicwall][af_policy_action]" } rename => { "[af_policy_object]" => "[logx][sonicwall][af_policy_object]" } rename => { "[active_interface]" => "[logx][sonicwall][active_interface]" } From ca1fda5d3414c43299a340d7fce56a9188034d31 Mon Sep 17 00:00:00 2001 From: JocLRojas Date: Mon, 27 Oct 2025 21:24:38 -0400 Subject: [PATCH 02/18] Upgrade pfSense filter to v2.1.0 with improved syslog parsing, optimized CSV processing for IPv4/IPv6 protocols and enhanced field handling --- filters/pfsense/pfsense_fw.conf | 450 ++++++++++++++++---------------- 1 file changed, 221 insertions(+), 229 deletions(-) diff --git a/filters/pfsense/pfsense_fw.conf b/filters/pfsense/pfsense_fw.conf index 47b1dd40a..cda03091c 100644 --- a/filters/pfsense/pfsense_fw.conf +++ b/filters/pfsense/pfsense_fw.conf @@ -1,258 +1,250 @@ filter { -# pfSense filter version 2.0.0 +# pfSense filter version 2.1.0 # Based on https://docs.netgate.com/pfsense/en/latest/monitoring/logs/raw-filter-format.html - split { - field => "message" - terminator => "" - } + split { + field => "message" + terminator => "" + } - #Looking for datasource generated by an agent and parse original message - if [message]=~/\[utm_stack_agent_ds=(.+)\]-(.+)/ { - grok { - match => { - "message" => [ "\[utm_stack_agent_ds=%{DATA:dataSource}\]-%{GREEDYDATA:original_log_message}" ] - } + #Looking for datasource generated by an agent and parse original message + if [message] =~ /\[utm_stack_agent_ds=(.+)\]-(.+)/ { + grok { + match => { + "message" => "\[utm_stack_agent_ds=%{DATA:dataSource}\]-%{GREEDYDATA:original_log_message}" + } + } } - } - if [original_log_message] { - mutate { - update => { "message" => "%{[original_log_message]}" } + if [original_log_message] { + mutate { + update => { "message" => "%{[original_log_message]}" + } + } } - } - if ![dataType] { + if ![dataType] { # The log destination is already identified by the agent so, don't need an entry point #......................................................................# #Generating dataType field required by CurrelationRulesEngine - mutate { - add_field => { "dataType" => "firewall-pfsense" } - } - -#......................................................................# -#Using grok to parse header of the message + mutate { + add_field => { "dataType" => "firewall-pfsense" } + } + + #......................................................................# + #Using grok to parse header of the message + grok { + match => { + "message" => [ + # Format with syslog header: version timestamp host program[pid]: data + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program}\[%{NUMBER:pid}\]: %{GREEDYDATA:msg_all}", + # Format with syslog header: version timestamp host program pid - - data + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program} %{NUMBER:pid} - - %{GREEDYDATA:msg_all}", + # Format with syslog header: version timestamp host program pid: data + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program} %{NUMBER:pid}: %{GREEDYDATA:msg_all}", + # Format with syslog header: version timestamp host program: data (no pid) + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program}: %{GREEDYDATA:msg_all}", + # Original pattern (legacy support): optional number at start + timestamp + "(%{INT:not_defined})?(\s)?(<%{NUMBER:priority}>)?(%{INT:syslog_version})? %{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{GREEDYDATA:msg_all}", + # Fallback: just timestamp and host for logs without syslog headers + "%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{GREEDYDATA:msg_all}", + # Direct CSV payload without any header + "%{IPORHOST:syslog_host} %{DATA:program}.*? %{GREEDYDATA:msg_all}" + ] + } + tag_on_failure => ["_grok_header_fail"] + } + + #......................................................................# + #Checking that the msg_all field exists + if [msg_all] { + if [message] =~ / filterlog(.+),(match|\w+),(block|pass),(in|out),(4|6)/ { grok { + match => { + "msg_all" => "%{WORD:event_type}(\s)?(\[)?(%{NUMBER:pid})?(\])?( - -|:) %{GREEDYDATA:csv_msg}" + } + } + #.....................................................................# + #Check if csv_msg exists and parsing it + if [csv_msg] { + #Changing the empty fields by X0X and then eliminating them + mutate { + gsub => [ + "csv_msg", ",,", ",X0X," + ] + } + #The gsub is repeated, because the first time it leaves some intermediate fields empty + mutate { + gsub => [ + "csv_msg", ",,", ",X0X," + ] + } + } + } else if [message] =~ / [a-z-_\.]+( \d+)? - - / { + grok { match => { - "message" => "(%{INT:not_defined})?(\s)?(<%{NUMBER:priority}>)?(%{INT:syslog_version})? %{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{GREEDYDATA:msg_all}" + "msg_all" => "%{DATA:event_type}( %{NUMBER:pid})? - - (- )?%{GREEDYDATA:msg}" } - } - - #......................................................................# - #Checking that the msg_all field exists - if ([msg_all]){ - if [message]=~/ filterlog(.+),(match|\w+),(block|pass),(in|out),(4|6)/{ - grok { - match => { - "msg_all" => "%{WORD:event_type}(\s)?(\[)?(%{NUMBER:pid})?(\])?( - -|:) %{GREEDYDATA:csv_msg}" - } - } - #.....................................................................# - #Check if csv_msg exists and parsing it - if [csv_msg]{ - #Changing the empty fields by X0X and then eliminating them - mutate { - gsub => [ - "csv_msg", ",,", ",X0X," - ] - } - #The gsub is repeated, because the first time it leaves some intermediate fields empty - mutate { - gsub => [ - "csv_msg", ",,", ",X0X," - ] - } - - if [message]=~/ filterlog(.+),(match|\w+),(block|pass),(in|out),4,(.+)(tcp|TCP|Tcp)/{ - csv { - source => "csv_msg" - skip_header => "true" - columns => ["rule_number", "sub_rule_number", "anchor", "tracker", "real_interface", "reason", - "action", "direction", "ip_version", "ipv4_tos", "ipv4_ecn", "ipv4_ttl", "ipv4_id", - "ipv4_offset", "ipv4_flags", "ipv4_protocol_id", "proto", "ip_length", "src_ip", "dest_ip", - "src_port", "dest_port", "data_length", "tcp_flags", "sequence_number", "ack_number", - "tcp_window", "urg", "tcp_options"] - } - }else if [message]=~/ filterlog(.+),(match|\w+),(block|pass),(in|out),4,(.+)(udp|UDP|Udp)/{ - csv { - source => "csv_msg" - skip_header => "true" - columns => ["rule_number", "sub_rule_number", "anchor", "tracker", "real_interface", "reason", - "action", "direction", "ip_version", "ipv4_tos", "ipv4_ecn", "ipv4_ttl", "ipv4_id", - "ipv4_offset", "ipv4_flags", "ipv4_protocol_id", "proto", "ip_length", "src_ip", "dest_ip", - "src_port", "dest_port", "data_length"] - } - } else if [message]=~/ filterlog(.+),(match|\w+),(block|pass),(in|out),4,(.+)(icmp|ICMP|Icmp)/{ - csv { - source => "csv_msg" - skip_header => "true" - columns => ["rule_number", "sub_rule_number", "anchor", "tracker", "real_interface", "reason", - "action", "direction", "ip_version", "ipv4_tos", "ipv4_ecn", "ipv4_ttl", "ipv4_id", - "ipv4_offset", "ipv4_flags", "ipv4_protocol_id", "proto", "ip_length", "src_ip", "dest_ip", - "icmp_type", "icmp_data1", "icmp_data2", "icmp_data3", "icmp_data4", "icmp_data5"] - - } - } else if [message]=~/ filterlog(.+),(match|\w+),(block|pass),(in|out),6,(.+)(tcp|TCP|Tcp)/{ - csv { - source => "csv_msg" - skip_header => "true" - columns => ["rule_number", "sub_rule_number", "anchor", "tracker", "real_interface", "reason", - "action", "direction", "ip_version", "ipv6_class", "ipv6_flow_label", "ipv6_hop_limit", "proto", - "ipv6_protocol_id", "ip_length", "src_ip", "dest_ip", - "src_port", "dest_port", "data_length", "tcp_flags", "sequence_number", "ack_number", - "tcp_window", "urg", "tcp_options"] - } - }else if [message]=~/ filterlog(.+),(match|\w+),(block|pass),(in|out),6,(.+)(udp|UDP|Udp)/{ - csv { - source => "csv_msg" - skip_header => "true" - columns => ["rule_number", "sub_rule_number", "anchor", "tracker", "real_interface", "reason", - "action", "direction", "ip_version", "ipv6_class", "ipv6_flow_label", "ipv6_hop_limit", "proto", - "ipv6_protocol_id", "ip_length", "src_ip", "dest_ip", - "src_port", "dest_port", "data_length"] - } - } else if [message]=~/ filterlog(.+),(match|\w+),(block|pass),(in|out),6,(.+)(icmp|ICMP|Icmp)/{ - csv { - source => "csv_msg" - skip_header => "true" - columns => ["rule_number", "sub_rule_number", "anchor", "tracker", "real_interface", "reason", - "action", "direction", "ip_version", "ipv6_class", "ipv6_flow_label", "ipv6_hop_limit", "proto", - "ipv6_protocol_id", "ip_length", "src_ip", "dest_ip", - "icmp_type", "icmp_data1", "icmp_data2", "icmp_data3", "icmp_data4", "icmp_data5"] - - } - } - #......................................................................# - #Rename the csv fields: - mutate { - rename => { "[rule_number]" => "[csv_field][rule_number]" } - rename => { "[sub_rule_number]" => "[csv_field][sub_rule_number]" } - rename => { "[anchor]" => "[csv_field][anchor]" } - rename => { "[tracker]" => "[csv_field][tracker]" } - rename => { "[real_interface]" => "[csv_field][real_interface]" } - rename => { "[reason]" => "[csv_field][reason]" } - rename => { "[action]" => "[csv_field][action]" } - rename => { "[direction]" => "[csv_field][direction]" } - rename => { "[ip_version]" => "[csv_field][ip_version]" } - rename => { "[ipv4_tos]" => "[csv_field][ipv4_tos]" } - rename => { "[ipv4_ecn]" => "[csv_field][ipv4_ecn]" } - rename => { "[ipv4_ttl]" => "[csv_field][ipv4_ttl]" } - rename => { "[ipv4_id]" => "[csv_field][ipv4_id]" } - rename => { "[ipv4_offset]" => "[csv_field][ipv4_offset]" } - rename => { "[ipv4_flags]" => "[csv_field][ipv4_flags]" } - rename => { "[ipv4_protocol_id]" => "[csv_field][ipv4_protocol_id]" } - rename => { "[proto]" => "[csv_field][proto]" } - rename => { "[ip_length]" => "[csv_field][ip_length]" } - rename => { "[src_ip]" => "[csv_field][src_ip]" } - rename => { "[dest_ip]" => "[csv_field][dest_ip]" } - rename => { "[src_port]" => "[csv_field][src_port]" } - rename => { "[dest_port]" => "[csv_field][dest_port]" } - rename => { "[data_length]" => "[csv_field][data_length]" } - rename => { "[tcp_flags]" => "[csv_field][tcp_flags]" } - rename => { "[sequence_number]" => "[csv_field][sequence_number]" } - rename => { "[ack_number]" => "[csv_field][ack_number]" } - rename => { "[tcp_window]" => "[csv_field][tcp_window]" } - rename => { "[urg]" => "[csv_field][urg]" } - rename => { "[tcp_options]" => "[csv_field][tcp_options]" } - rename => { "[ipv6_class]" => "[csv_field][ipv6_class]" } - rename => { "[ipv6_flow_label]" => "[csv_field][ipv6_flow_label]" } - rename => { "[ipv6_hop_limit]" => "[csv_field][ipv6_hop_limit]" } - rename => { "[ipv6_protocol_id]" => "[csv_field][ipv6_protocol_id]" } - rename => { "[icmp_type]" => "[csv_field][icmp_type]" } - rename => { "[icmp_data1]" => "[csv_field][icmp_data1]" } - rename => { "[icmp_data2]" => "[csv_field][icmp_data2]" } - rename => { "[icmp_data3]" => "[csv_field][icmp_data3]" } - rename => { "[icmp_data4]" => "[csv_field][icmp_data4]" } - rename => { "[icmp_data5]" => "[csv_field][icmp_data5]" } - } - - #......................................................................# - # Delete empty csv fields with ruby code - if [csv_field] { - ruby { - code => ' - event.get("[csv_field]").each do |k, v| - if (v == "X0X") - event.remove(k) - else - event.set("[logx][pfsense][#{k}]",v) - end - end - ' - } - } - } - }else if[message]=~/ [a-z-_\.]+( \d+)? - - /{ - grok { - match => { - "msg_all" => "%{DATA:event_type}( %{NUMBER:pid})? - - (- )?%{GREEDYDATA:msg}" - } - } - }else if [message]=~/ \/(.+?)( \d+)? - - \((.+?)\) [A-Z]+ \((.+?)\)/{ - grok { - match => { - "msg_all" => "%{PATH:process}( %{NUMBER:pid})? - - \(%{DATA:user}\) %{WORD:command_action} \(%{PATH:process_path}\)(GREEDYDATA:)?" - } - } + } + } else if [message] =~ /\/(.+?)( \d+)? - - \((.+?)\) [A-Z]+ \((.+?)\)/ { + grok { + match => { + "msg_all" => "%{PATH:process}( %{NUMBER:pid})? - - \(%{DATA:user}\) %{WORD:command_action} \(%{PATH:process_path}\)" } + } + } + } + #......................................................................# + # Rename other fields + mutate { + rename => { + "event_type" => "[logx][pfsense][event_type]" + "msg" => "[logx][pfsense][msg]" + "pid" => "[logx][pfsense][pid]" + "priority" => "[logx][pfsense][priority]" + "message" => "[logx][pfsense][message]" + } + } + #.....................................................................# + #Generating dataSource field required by Correlation Engine + if [syslog_host] { + mutate { + rename => { "syslog_host" => "[logx][pfsense][syslog_host]" } + update => { "dataSource" => "%{[logx][pfsense][syslog_host]}" } + } + } + if (![dataSource]){ + mutate { + add_field => { "dataSource" => "%{host}" } } + } - #......................................................................# - # Rename other fields - mutate { - rename => { "[event_type]" => "[logx][pfsense][event_type]" } - rename => { "[msg]" => "[logx][pfsense][msg]" } - rename => { "[pid]" => "[logx][pfsense][pid]" } - rename => { "[priority]" => "[logx][pfsense][priority]" } - rename => { "[user]" => "[logx][pfsense][user]" } - rename => { "[cmd]" => "[logx][pfsense][cmd]" } - rename => { "[process]" => "[logx][pfsense][process]" } - rename => { "[command_action]" => "[logx][pfsense][command_action]" } - rename => { "[process_path]" => "[logx][pfsense][process_path]" } - rename => { "[message]" => "[logx][pfsense][message]" } + if [logx][pfsense][message] and [logx][pfsense][message] =~ /^ { + "[logx][pfsense][message]" => [ + "<%{NUMBER:priority}>%{INT:syslog_version} %{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}(?: %{NUMBER:pid})? - - %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}\[%{NUMBER:pid}\]: %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program} %{NUMBER:pid} - - %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program} %{NUMBER:pid}: %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}: %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}.*? %{GREEDYDATA:payload}" + ] } - - #.....................................................................# - #Generating dataSource field required by Correlation Engine - if [syslog_host]{ - mutate { - update => { "[dataSource]" => "%{[syslog_host]}" } - } - mutate { - rename => { "[syslog_host]" => "[logx][pfsense][syslog_host]" } - } + tag_on_failure => ["_grok2_fail"] + } + if !("_grok2_fail" in [tags]) and [payload] { + mutate { + replace => { "[logx][pfsense][message]" => "%{[payload]}" } + } } - if (![dataSource]){ - mutate { - add_field => { "dataSource" => "%{host}" } - } + } + #......................................................................# + #Remove unnecessary fields + mutate { + remove_field => [ + "@version","timestamp","payload","deviceTime","path","type","syslog_version","syslog_host","not_defined", + "pid","priority","program","event","csv_msg","csv_field","msg_all","original_log_message","tags","headers" + ] + } + } + + if [logx][pfsense][message] { + + #......................................................................# + #IPv4 branches + if [logx][pfsense][message] =~ /,4,/ { + + #......................................................................# + #IPv4 ICMP: map ICMP-specific tail + if [logx][pfsense][message] =~ /,icmp,/ { + csv { + source => "[logx][pfsense][message]" + target => "[logx][pfsense]" + separator => "," + skip_empty_columns => true + columns => [ + "rule_number","sub_rule_number","anchor","tracker","interface","reason", + "action","direction","ip_version","ipv4_tos","ipv4_ecn","ipv4_ttl", + "ipv4_id","ipv4_offset","ipv4_flags","ipv4_protocol_id","proto","ip_length", + "src_ip","dest_ip", + "icmp_type","icmp_data1","icmp_data2","icmp_data3","icmp_data4","icmp_data5" + ] } - #......................................................................# - #Generating action field for established connections - if [logx][pfsense][action] and [logx][pfsense][action] == "pass" { - mutate { - add_field => { "[logx][utm][action]" => "Success" } - } + #......................................................................# + #IPv4 TCP/UDP: share list; TCP fills extra fields, UDP leaves them nil + } else { + csv { + source => "[logx][pfsense][message]" + target => "[logx][pfsense]" + separator => "," + skip_empty_columns => true + columns => [ + "rule_number","sub_rule_number","anchor","tracker","interface","reason", + "action","direction","ip_version","ipv4_tos","ipv4_ecn","ipv4_ttl", + "ipv4_id","ipv4_offset","ipv4_flags","ipv4_protocol_id","proto","ip_length", + "src_ip","dest_ip", + "src_port","dest_port","data_length", + "tcp_flags","sequence_number","ack_number","tcp_window","urg","tcp_options" + ] } - if [logx][pfsense][msg] and [logx][pfsense][msg]=~/[Cc]onnection reset/{ - mutate { - add_field => { "[logx][utm][action]" => "Success" } - } + } + } + + #......................................................................# + #IPv6 branches + else if [logx][pfsense][message] =~ /,6,/ { + + #......................................................................# + #IPv6 ICMP + if [logx][pfsense][message] =~ /,icmp,/ { + csv { + source => "[logx][pfsense][message]" + target => "[logx][pfsense]" + separator => "," + skip_empty_columns => true + columns => [ + "rule_number","sub_rule_number","anchor","tracker","interface","reason", + "action","direction","ip_version","ipv6_class","ipv6_flow_label","ipv6_hop_limit", + "proto","ipv6_protocol_id","ip_length","src_ip","dest_ip", + "icmp_type","icmp_data1","icmp_data2","icmp_data3","icmp_data4","icmp_data5" + ] } - #......................................................................# - #Finally, remove unnecessary fields - mutate { - remove_field => ["@version","timestamp","path","type","syslog_version", - "not_defined", "path", "event", "csv_msg", "csv_field", "msg_all"] + #......................................................................# + # IPv6 TCP/UDP + } else { + csv { + source => "[logx][pfsense][message]" + target => "[logx][pfsense]" + separator => "," + skip_empty_columns => true + columns => [ + "rule_number","sub_rule_number","anchor","tracker","interface","reason", + "action","direction","ip_version","ipv6_class","ipv6_flow_label","ipv6_hop_limit", + "proto","ipv6_protocol_id","ip_length","src_ip","dest_ip", + "src_port","dest_port","data_length", + "tcp_flags","sequence_number","ack_number","tcp_window","urg","tcp_options" + ] } + } + } - } - #Also, remove unwanted fields if the message not match with conditions - mutate { - remove_field => ["@version","path","original_log_message","headers", "tags"] - } + #......................................................................# + #Generating action field for established connections + if [logx][pfsense][action] == "pass" { + mutate { add_field => { "[logx][utm][action]" => "Success" } } + } + } + mutate { + remove_field => "[logx][pfsense][message]" + } } From 5a12f0325383ba13f0a91668df57aa135fc84e51 Mon Sep 17 00:00:00 2001 From: Yadian Llada Lopez Date: Fri, 31 Oct 2025 08:45:50 -0400 Subject: [PATCH 03/18] feat(agent): implement TLS support for syslog integrations. --- agent/config/const.go | 6 ++ agent/main.go | 88 ++++++++++++++++- agent/modules/configuration.go | 124 +++++++++++++++++++++++- agent/modules/modules.go | 9 +- agent/modules/netflow.go | 12 ++- agent/modules/syslog.go | 127 +++++++++++++++++++++++-- agent/utils/files.go | 17 ++++ agent/utils/int_tls.go | 169 +++++++++++++++++++++++++++++++++ 8 files changed, 531 insertions(+), 21 deletions(-) create mode 100644 agent/utils/int_tls.go diff --git a/agent/config/const.go b/agent/config/const.go index 10bc21961..94960361f 100644 --- a/agent/config/const.go +++ b/agent/config/const.go @@ -36,6 +36,12 @@ var ( MESSAGE_HEADER = "utm_stack_agent_ds" BatchCapacity = 100 + // TLS Configuration for Integrations + IntegrationCertPath = filepath.Join(utils.GetMyPath(), "certs", "integration.crt") + IntegrationKeyPath = filepath.Join(utils.GetMyPath(), "certs", "integration.key") + IntegrationCAPath = filepath.Join(utils.GetMyPath(), "certs", "integration-ca.crt") + + // MaxConnectionTime = 120 * time.Second // SERV_NAME = "UTMStackAgent" // SERV_LOG = "utmstack_agent.log" diff --git a/agent/main.go b/agent/main.go index 57bf702d4..3e481a669 100644 --- a/agent/main.go +++ b/agent/main.go @@ -84,18 +84,88 @@ func main() { serv.InstallService() fmt.Println("[OK]") fmt.Println("UTMStackAgent service installed correctly") + case "enable-integration", "disable-integration": fmt.Println("Changing integration status ...") integration := os.Args[2] proto := os.Args[3] - port, err := modules.ChangeIntegrationStatus(integration, proto, (arg == "enable-integration")) + tlsEnabled := false + for _, arg := range os.Args[4:] { + if arg == "--tls" { + tlsEnabled = true + break + } + } + + var port string + var err error + + if arg == "enable-integration" && tlsEnabled { + port, err = modules.ChangeIntegrationStatus(integration, proto, true, true) + } else if arg == "enable-integration" { + port, err = modules.ChangeIntegrationStatus(integration, proto, true, false) + } else { + port, err = modules.ChangeIntegrationStatus(integration, proto, false) + } + if err != nil { - fmt.Println("Error trying to change integration status: ", err) + fmt.Println("Error:", err) + os.Exit(1) + } + + if arg == "enable-integration" && tlsEnabled { + fmt.Printf("Integration %s %s enabled with TLS on port %s\n", integration, proto, port) + } else if arg == "enable-integration" { + fmt.Printf("Integration %s %s enabled on port %s\n", integration, proto, port) + } else { + fmt.Printf("Integration %s %s disabled (port %s freed)\n", integration, proto, port) + } + time.Sleep(5 * time.Second) + + case "load-tls-certs": + if len(os.Args) < 4 { + fmt.Println("Usage: ./utmstack_agent load-tls-certs [ca_certificate_path]") + fmt.Println("Example: ./utmstack_agent load-tls-certs /path/to/server.crt /path/to/server.key /path/to/ca.crt") + os.Exit(1) + } + + userCertPath := os.Args[2] + userKeyPath := os.Args[3] + var userCAPath string + if len(os.Args) > 4 { + userCAPath = os.Args[4] + } + + fmt.Println("Loading user TLS certificates ...") + + fmt.Print("Validating certificate files ... ") + if err := utils.ValidateIntegrationCertificates(userCertPath, userKeyPath); err != nil { + fmt.Printf("\nError: Invalid certificate files: %v\n", err) os.Exit(1) } - fmt.Printf("Action %s %s %s correctly in port %s\n", arg, integration, proto, port) + fmt.Println("[OK]") + + fmt.Print("Installing certificates ... ") + src := utils.CertificateFiles{ + CertPath: userCertPath, + KeyPath: userKeyPath, + CAPath: userCAPath, + } + dest := utils.CertificateFiles{ + CertPath: config.IntegrationCertPath, + KeyPath: config.IntegrationKeyPath, + CAPath: config.IntegrationCAPath, + } + if err := utils.LoadUserCertificatesWithStruct(src, dest); err != nil { + fmt.Printf("\nError loading certificates: %v\n", err) + os.Exit(1) + } + fmt.Println("[OK]") + + fmt.Println("TLS certificates loaded successfully!") time.Sleep(5 * time.Second) + case "change-port": fmt.Println("Changing integration port ...") integration := os.Args[2] @@ -109,6 +179,7 @@ func main() { } fmt.Printf("Port changed correctly from %s to %s\n", old, port) time.Sleep(5 * time.Second) + case "uninstall": fmt.Print("Uninstalling UTMStackAgent service ...") @@ -160,6 +231,17 @@ func Help() { fmt.Println(" uninstall Uninstall the UTMStackAgent service") fmt.Println(" help Display this help message") fmt.Println() + fmt.Println("TLS Certificate Management:") + fmt.Println(" # Load your own certificates (RECOMMENDED)") + fmt.Println(" ./utmstack_agent load-tls-certs /path/to/server.crt /path/to/server.key /path/to/ca.crt") + fmt.Println(" ./utmstack_agent load-tls-certs /path/to/server.crt /path/to/server.key # Without CA") + fmt.Println() + fmt.Println("TLS Integration Examples:") + fmt.Println(" ./utmstack_agent enable-integration syslog tcp --tls # Enable with TLS") + fmt.Println(" ./utmstack_agent enable-integration syslog tcp # Enable without TLS (default)") + fmt.Println(" ./utmstack_agent disable-integration syslog tcp # Disable (auto-disables TLS)") + fmt.Println(" ./utmstack_agent check-tls-certs # Check certificate status") + fmt.Println() fmt.Println("Note:") fmt.Println(" - Make sure to run commands with appropriate permissions.") fmt.Println(" - All commands require administrative privileges.") diff --git a/agent/modules/configuration.go b/agent/modules/configuration.go index bbf527d32..ba55f856a 100644 --- a/agent/modules/configuration.go +++ b/agent/modules/configuration.go @@ -5,14 +5,16 @@ import ( "net" "os" "strings" + "time" "github.com/utmstack/UTMStack/agent/config" "github.com/utmstack/UTMStack/agent/utils" ) type Port struct { - IsListen bool `json:"enabled"` - Port string `json:"value"` + IsListen bool `json:"enabled"` + Port string `json:"value"` + TLSEnabled bool `json:"tls_enabled,omitempty"` } type Integration struct { @@ -59,7 +61,7 @@ func ConfigureCollectorFirstTime() error { return WriteCollectorConfig(integrations, config.CollectorFileName) } -func ChangeIntegrationStatus(logTyp string, proto string, isEnabled bool) (string, error) { +func ChangeIntegrationStatus(logTyp string, proto string, isEnabled bool, tlsOptions ...bool) (string, error) { var port string cnf, err := ReadCollectorConfig() if err != nil { @@ -78,9 +80,49 @@ func ChangeIntegrationStatus(logTyp string, proto string, isEnabled bool) (strin case "tcp": integration.TCP.IsListen = isEnabled port = integration.TCP.Port + + // Handle TLS configuration if specified + if len(tlsOptions) > 0 && isEnabled { + if tlsOptions[0] { + // Enable TLS + integration.TCP.TLSEnabled = true + mod := GetModule(logTyp) + if mod != nil && mod.IsPortListen(proto) { + mod.DisablePort(proto) + time.Sleep(100 * time.Millisecond) + err := mod.EnablePort(proto, true) + if err != nil { + return "", fmt.Errorf("error enabling TLS on running module: %v", err) + } + } + } else { + // Disable TLS + integration.TCP.TLSEnabled = false + mod := GetModule(logTyp) + if mod != nil && mod.IsPortListen(proto) { + mod.DisablePort(proto) + time.Sleep(100 * time.Millisecond) + err := mod.EnablePort(proto, false) + if err != nil { + return "", fmt.Errorf("error disabling TLS on running module: %v", err) + } + } + } + } + + // Auto-disable TLS when disabling integration + if !isEnabled { + integration.TCP.TLSEnabled = false + } + case "udp": integration.UDP.IsListen = isEnabled port = integration.UDP.Port + + // TLS validation for UDP + if len(tlsOptions) > 0 && tlsOptions[0] { + return "", fmt.Errorf("TLS is not supported for UDP protocol. Use TCP for TLS connections") + } } cnf.Integrations[logTyp] = integration @@ -145,7 +187,11 @@ func WriteCollectorConfig(integrations map[string]Integration, filename string) for name, integration := range integrations { fileContent += fmt.Sprintf(" \"%s\": {\n", name) if integration.TCP.Port != "" { - fileContent += fmt.Sprintf(" \"tcp_port\": {\"enabled\": %t, \"value\": \"%s\"},\n", integration.TCP.IsListen, integration.TCP.Port) + fileContent += fmt.Sprintf(" \"tcp_port\": {\"enabled\": %t, \"value\": \"%s\"", integration.TCP.IsListen, integration.TCP.Port) + if integration.TCP.TLSEnabled { + fileContent += fmt.Sprintf(", \"tls_enabled\": %t", integration.TCP.TLSEnabled) + } + fileContent += "},\n" } if integration.UDP.Port != "" { fileContent += fmt.Sprintf(" \"udp_port\": {\"enabled\": %t, \"value\": \"%s\"},\n", integration.UDP.IsListen, integration.UDP.Port) @@ -184,3 +230,73 @@ func WriteCollectorConfigFromModules(mod []Module, filename string) error { } return WriteCollectorConfig(integrations, filename) } + +func EnableTLSForIntegration(logTyp string, proto string) (string, error) { + cnf, err := ReadCollectorConfig() + if err != nil { + return "", fmt.Errorf("error reading collector config: %v", err) + } + + if valid := config.ValidateModuleType(logTyp); valid == "nil" { + return "", fmt.Errorf("invalid integration: %s", logTyp) + } + + integration := cnf.Integrations[logTyp] + var port string + + switch proto { + case "tcp": + if integration.TCP.Port == "" { + return "", fmt.Errorf("TCP port not configured for %s", logTyp) + } + port = integration.TCP.Port + integration.TCP.TLSEnabled = true + + mod := GetModule(logTyp) + if mod != nil && mod.IsPortListen(proto) { + mod.DisablePort(proto) + time.Sleep(100 * time.Millisecond) + err := mod.EnablePort(proto, true) + if err != nil { + return port, fmt.Errorf("error enabling TLS on running module: %v", err) + } + } + case "udp": + return "", fmt.Errorf("TLS not supported for UDP protocol") + default: + return "", fmt.Errorf("invalid protocol: %s", proto) + } + + cnf.Integrations[logTyp] = integration + return port, WriteCollectorConfig(cnf.Integrations, config.CollectorFileName) +} + +func DisableTLSForIntegration(logTyp string, proto string) error { + cnf, err := ReadCollectorConfig() + if err != nil { + return fmt.Errorf("error reading collector config: %v", err) + } + + integration := cnf.Integrations[logTyp] + switch proto { + case "tcp": + integration.TCP.TLSEnabled = false + + mod := GetModule(logTyp) + if mod != nil && mod.IsPortListen(proto) { + mod.DisablePort(proto) + time.Sleep(100 * time.Millisecond) + err := mod.EnablePort(proto, false) + if err != nil { + return fmt.Errorf("error disabling TLS on running module: %v", err) + } + } + case "udp": + return fmt.Errorf("TLS not supported for UDP protocol") + default: + return fmt.Errorf("invalid protocol: %s", proto) + } + + cnf.Integrations[logTyp] = integration + return WriteCollectorConfig(cnf.Integrations, config.CollectorFileName) +} diff --git a/agent/modules/modules.go b/agent/modules/modules.go index 508f2cecc..a108e549a 100644 --- a/agent/modules/modules.go +++ b/agent/modules/modules.go @@ -20,7 +20,7 @@ type Module interface { IsPortListen(proto string) bool SetNewPort(proto string, port string) GetPort(proto string) string - EnablePort(proto string) + EnablePort(proto string, enableTLS bool) error DisablePort(proto string) } @@ -88,7 +88,12 @@ func StartModules() { if changeAllowed { moCache[index].SetNewPort(proto, port) if conf[1] { - moCache[index].EnablePort(proto) + enableTLS := proto == "tcp" && cnf.TCP.TLSEnabled + + err := moCache[index].EnablePort(proto, enableTLS) + if err != nil { + utils.Logger.ErrorF("error enabling port for %s %s: %v", intType, proto, err) + } } } else { utils.Logger.Info("change in port %s protocol %s not allowed for %s or out range %s-%s", port, proto, intType, config.PortRangeMin, config.PortRangeMax) diff --git a/agent/modules/netflow.go b/agent/modules/netflow.go index c2dd6ca16..0b4cb30ae 100644 --- a/agent/modules/netflow.go +++ b/agent/modules/netflow.go @@ -45,15 +45,20 @@ func GetNetflowModule() *NetflowModule { return netflowModule } -func (m *NetflowModule) EnablePort(proto string) { +func (m *NetflowModule) EnablePort(proto string, enableTLS bool) error { if proto == "udp" && !m.IsEnabled { + // NetFlow over UDP doesn't support TLS + if enableTLS { + utils.Logger.Info("TLS not supported for NetFlow (UDP protocol), ignoring TLS flag") + } + utils.Logger.Info("Server %s listening in port: %s protocol: UDP", m.DataType, config.ProtoPorts[config.DataTypeNetflow].UDP) m.IsEnabled = true port, err := strconv.Atoi(config.ProtoPorts[config.DataTypeNetflow].UDP) if err != nil { utils.Logger.ErrorF("error converting port to int: %v", err) - return + return err } m.Listener, err = net.ListenUDP("udp", &net.UDPAddr{ @@ -62,7 +67,7 @@ func (m *NetflowModule) EnablePort(proto string) { }) if err != nil { utils.Logger.ErrorF("error listening netflow: %v", err) - return + return err } m.CTX, m.Cancel = context.WithCancel(context.Background()) @@ -122,6 +127,7 @@ func (m *NetflowModule) EnablePort(proto string) { } }() } + return nil } func (m *NetflowModule) DisablePort(proto string) { diff --git a/agent/modules/syslog.go b/agent/modules/syslog.go index cdb977d1f..4c02fd5aa 100644 --- a/agent/modules/syslog.go +++ b/agent/modules/syslog.go @@ -3,6 +3,7 @@ package modules import ( "bufio" "context" + "crypto/tls" "errors" "io" "net" @@ -25,11 +26,12 @@ type SyslogModule struct { } type listenerTCP struct { - Listener net.Listener - CTX context.Context - Cancel context.CancelFunc - IsEnabled bool - Port string + Listener net.Listener + CTX context.Context + Cancel context.CancelFunc + IsEnabled bool + Port string + TLSEnabled bool } type listenerUDP struct { @@ -91,13 +93,20 @@ func (m *SyslogModule) GetPort(proto string) string { } } -func (m *SyslogModule) EnablePort(proto string) { +func (m *SyslogModule) EnablePort(proto string, enableTLS bool) error { switch proto { case "tcp": + // Update TLS configuration before enabling + m.TCPListener.TLSEnabled = enableTLS go m.enableTCP() case "udp": + // UDP doesn't support TLS, log warning if TLS is requested + if enableTLS { + utils.Logger.Info("TLS not supported for UDP protocol in %s, ignoring TLS flag", m.DataType) + } go m.enableUDP() } + return nil } func (m *SyslogModule) DisablePort(proto string) { @@ -156,7 +165,13 @@ func (m *SyslogModule) enableTCP() { utils.Logger.ErrorF("error connecting with tcp listener: %v", err) continue } - go m.handleConnectionTCP(conn) + + // Connection handling based on TLS configuration + if m.TCPListener.TLSEnabled { + go m.handleTLSConnection(conn) + } else { + go m.handleConnectionTCP(conn) + } } } }() @@ -242,8 +257,14 @@ func (m *SyslogModule) enableUDP() { func (m *SyslogModule) disableTCP() { if m.TCPListener.IsEnabled && m.TCPListener.Port != "" { utils.Logger.Info("Server %s closed in port: %s protocol: TCP", m.DataType, m.TCPListener.Port) + + if m.TCPListener.Listener != nil { + if err := m.TCPListener.Listener.Close(); err != nil { + utils.Logger.ErrorF("error closing TCP listener: %v", err) + } + } + m.TCPListener.Cancel() - m.TCPListener.Listener.Close() m.TCPListener.IsEnabled = false } } @@ -251,8 +272,14 @@ func (m *SyslogModule) disableTCP() { func (m *SyslogModule) disableUDP() { if m.UDPListener.IsEnabled && m.UDPListener.Port != "" { utils.Logger.Info("Server %s closed in port: %s protocol: UDP", m.DataType, m.UDPListener.Port) + + if m.UDPListener.Listener != nil { + if err := m.UDPListener.Listener.Close(); err != nil { + utils.Logger.ErrorF("error closing UDP listener: %v", err) + } + } + m.UDPListener.Cancel() - m.UDPListener.Listener.Close() m.UDPListener.IsEnabled = false } } @@ -275,6 +302,25 @@ func (m *SyslogModule) handleConnectionTCP(c net.Conn) { } } + // Detect and reject TLS connections when TLS is disabled + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + firstBytes := make([]byte, 3) + n, err := reader.Read(firstBytes) + if err != nil { + utils.Logger.ErrorF("error reading initial bytes from %s: %v", remoteAddr, err) + return + } + + // TLS handshake starts with: 0x16 (22 decimal) for TLS 1.0-1.3 + if n >= 1 && firstBytes[0] == 0x16 { + utils.Logger.ErrorF("TLS connection rejected from %s: TLS is disabled, only plain text connections accepted", remoteAddr) + return + } + + // Reset deadline and create a new reader that includes the read bytes + c.SetReadDeadline(time.Time{}) + reader = bufio.NewReader(io.MultiReader(strings.NewReader(string(firstBytes[:n])), reader)) + msgChannel := make(chan string) go m.handleMessageTCP(msgChannel) @@ -297,6 +343,69 @@ func (m *SyslogModule) handleConnectionTCP(c net.Conn) { } } +func (m *SyslogModule) handleTLSConnection(conn net.Conn) { + defer conn.Close() + + remoteAddr := conn.RemoteAddr().String() + remoteAddr, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + utils.Logger.ErrorF("error splitting host and port: %v", err) + remoteAddr = "unknown" + } + + if remoteAddr == "127.0.0.1" { + if hostname, err := os.Hostname(); err == nil { + remoteAddr = hostname + } + } + + tlsConfig, err := utils.LoadIntegrationTLSConfig( + config.IntegrationCertPath, + config.IntegrationKeyPath, + ) + if err != nil { + utils.Logger.ErrorF("error loading TLS config: %v", err) + return + } + + tlsConn := tls.Server(conn, tlsConfig) + + conn.SetDeadline(time.Now().Add(10 * time.Second)) + if err := tlsConn.Handshake(); err != nil { + utils.Logger.ErrorF("TLS handshake failed from %s: %v", remoteAddr, err) + return + } + // Keep a reasonable read timeout instead of removing it entirely + conn.SetDeadline(time.Now().Add(30 * time.Second)) + + reader := bufio.NewReader(tlsConn) + msgChannel := make(chan string) + go m.handleMessageTCP(msgChannel) + + for { + select { + case <-m.TCPListener.CTX.Done(): + return + default: + // Set read timeout for each message + conn.SetDeadline(time.Now().Add(30 * time.Second)) + message, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + return + } + if netErr, ok := err.(net.Error); ok && netErr.Timeout() { + return + } + utils.Logger.ErrorF("error reading TLS data from %s: %v", remoteAddr, err) + return + } + message = config.GetMessageFormated(remoteAddr, message) + msgChannel <- message + } + } +} + func (m *SyslogModule) handleMessageTCP(logsChannel chan string) { logBatch := []string{} ticker := time.NewTicker(5 * time.Second) diff --git a/agent/utils/files.go b/agent/utils/files.go index 977b5689c..51da99255 100644 --- a/agent/utils/files.go +++ b/agent/utils/files.go @@ -157,3 +157,20 @@ func IsDirEmpty(path string) (bool, error) { } return false, err } + +func copyFile(src, dst string) error { + sourceFile, err := os.Open(src) + if err != nil { + return err + } + defer sourceFile.Close() + + destFile, err := os.Create(dst) + if err != nil { + return err + } + defer destFile.Close() + + _, err = io.Copy(destFile, sourceFile) + return err +} diff --git a/agent/utils/int_tls.go b/agent/utils/int_tls.go new file mode 100644 index 000000000..5ca5928f3 --- /dev/null +++ b/agent/utils/int_tls.go @@ -0,0 +1,169 @@ +package utils + +import ( + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "fmt" + "os" + "path/filepath" + "time" +) + +const ( + CertFilePermissions = 0644 + KeyFilePermissions = 0600 + MinTLSVersion = tls.VersionTLS12 + MaxTLSVersion = tls.VersionTLS13 +) + +type TLSStatus struct { + Available bool `json:"available"` + CertExists bool `json:"cert_exists"` + KeyExists bool `json:"key_exists"` + CAExists bool `json:"ca_exists"` + Valid bool `json:"valid"` + Error string `json:"error,omitempty"` +} + +type CertificateFiles struct { + CertPath string + KeyPath string + CAPath string +} + +func LoadIntegrationTLSConfig(certPath, keyPath string) (*tls.Config, error) { + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return nil, fmt.Errorf("error loading TLS certificate: %w", err) + } + + return &tls.Config{ + Certificates: []tls.Certificate{cert}, + MinVersion: MinTLSVersion, + MaxVersion: MaxTLSVersion, + CipherSuites: []uint16{ + // TLS 1.2 secure cipher suites - RSA key exchange + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + // TLS 1.2 secure cipher suites - ECDSA key exchange (for ECDSA certificates) + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + }, + CurvePreferences: []tls.CurveID{ + tls.X25519, // Modern and fast + tls.CurveP256, // NIST P-256 + tls.CurveP384, // NIST P-384 + tls.CurveP521, // NIST P-521 + }, + PreferServerCipherSuites: true, + }, nil +} + +func ValidateIntegrationCertificates(certPath, keyPath string) error { + if !CheckIfPathExist(certPath) { + return fmt.Errorf("certificate file not found: %s", certPath) + } + + if !CheckIfPathExist(keyPath) { + return fmt.Errorf("private key file not found: %s", keyPath) + } + + cert, err := tls.LoadX509KeyPair(certPath, keyPath) + if err != nil { + return fmt.Errorf("invalid certificate or private key: %w", err) + } + + x509Cert, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return fmt.Errorf("error parsing certificate: %w", err) + } + + // 1. Check validity dates + now := time.Now() + if now.Before(x509Cert.NotBefore) { + return fmt.Errorf("certificate is not yet valid (valid from: %s)", + x509Cert.NotBefore.Format("2006-01-02 15:04:05 UTC")) + } + + if now.After(x509Cert.NotAfter) { + return fmt.Errorf("certificate has expired (valid until: %s)", + x509Cert.NotAfter.Format("2006-01-02 15:04:05 UTC")) + } + + // 2. Warn if the certificate expires soon (30 days) + if now.Add(30 * 24 * time.Hour).After(x509Cert.NotAfter) { + fmt.Printf("WARNING: Certificate expires soon (%s)\n", + x509Cert.NotAfter.Format("2006-01-02 15:04:05 UTC")) + } + + // 3. Check signature algorithm (reject weak algorithms) + switch x509Cert.SignatureAlgorithm { + case x509.SHA1WithRSA, x509.MD5WithRSA: + return fmt.Errorf("certificate uses weak signature algorithm: %s (use SHA256+ instead)", + x509Cert.SignatureAlgorithm) + } + + // 4. Check RSA key size (minimum 2048 bits) + if x509Cert.PublicKeyAlgorithm == x509.RSA { + if rsaKey, ok := x509Cert.PublicKey.(*rsa.PublicKey); ok { + keySize := rsaKey.Size() * 8 // Convert bytes to bits + if keySize < 2048 { + return fmt.Errorf("RSA key size too small: %d bits (minimum 2048 bits required)", keySize) + } + } + } + + return nil +} + +func LoadUserCertificatesWithStruct(src, dest CertificateFiles) error { + // Validate source certificates + if !CheckIfPathExist(src.CertPath) { + return fmt.Errorf("user certificate file not found: %s", src.CertPath) + } + if !CheckIfPathExist(src.KeyPath) { + return fmt.Errorf("user private key file not found: %s", src.KeyPath) + } + if err := ValidateIntegrationCertificates(src.CertPath, src.KeyPath); err != nil { + return err + } + + // Prepare destination directory + certsDir := filepath.Dir(dest.CertPath) + if err := CreatePathIfNotExist(certsDir); err != nil { + return fmt.Errorf("error creating certificates directory: %w", err) + } + + // Copy certificate files + if err := copyFile(src.CertPath, dest.CertPath); err != nil { + return fmt.Errorf("error copying certificate: %w", err) + } + if err := copyFile(src.KeyPath, dest.KeyPath); err != nil { + return fmt.Errorf("error copying private key: %w", err) + } + + // Copy CA certificate (use source CA if exists, otherwise use cert as CA) + caSource := src.CAPath + if caSource == "" || !CheckIfPathExist(caSource) { + caSource = src.CertPath + } + if err := copyFile(caSource, dest.CAPath); err != nil { + return fmt.Errorf("error copying CA certificate: %w", err) + } + + // Set file permissions + if err := os.Chmod(dest.CertPath, CertFilePermissions); err != nil { + return fmt.Errorf("error setting certificate permissions: %w", err) + } + if err := os.Chmod(dest.KeyPath, KeyFilePermissions); err != nil { + return fmt.Errorf("error setting private key permissions: %w", err) + } + if err := os.Chmod(dest.CAPath, CertFilePermissions); err != nil { + return fmt.Errorf("error setting CA permissions: %w", err) + } + + return nil +} From 142e593f5b460fe21e89165c31af483079c02693 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Fri, 31 Oct 2025 09:15:04 -0500 Subject: [PATCH 04/18] fix(account): sanitize password in checkPassword method to prevent injection Signed-off-by: Manuel Abascal --- frontend/src/app/core/auth/account.service.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/core/auth/account.service.ts b/frontend/src/app/core/auth/account.service.ts index e394efcd2..b2bb245cd 100644 --- a/frontend/src/app/core/auth/account.service.ts +++ b/frontend/src/app/core/auth/account.service.ts @@ -25,7 +25,8 @@ export class AccountService { } checkPassword(password: string, uuid: string): Observable> { - return this.http.get(SERVER_API_URL + `api/check-credentials?password=${password}&checkUUID=${uuid}`, { + const sanitizedPassword = encodeURIComponent(password); + return this.http.get(SERVER_API_URL + `api/check-credentials?password=${sanitizedPassword}&checkUUID=${uuid}`, { observe: 'response', responseType: 'text' }); From b28c6d555f439bb2ce889c6c5e348a3152abb772 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Fri, 31 Oct 2025 09:15:15 -0500 Subject: [PATCH 05/18] feat(date-timezone): add New Zealand and Fiji timezones to the constants Signed-off-by: Manuel Abascal --- frontend/src/app/shared/constants/date-timezone-date.const.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/frontend/src/app/shared/constants/date-timezone-date.const.ts b/frontend/src/app/shared/constants/date-timezone-date.const.ts index ac2068658..cd73ab06c 100644 --- a/frontend/src/app/shared/constants/date-timezone-date.const.ts +++ b/frontend/src/app/shared/constants/date-timezone-date.const.ts @@ -25,6 +25,8 @@ export const TIMEZONES: Array<{ label: string; timezone: string, zone: string }> {label: 'Sydney (AEST)', timezone: 'Australia/Sydney', zone: 'Australia'}, {label: 'Melbourne (AEST)', timezone: 'Australia/Melbourne', zone: 'Australia'}, {label: 'Perth (AWST)', timezone: 'Australia/Perth', zone: 'Australia'}, + {label: 'New Zealand (NZST)', timezone: 'Pacific/Auckland', zone: 'Pacific'}, + {label: 'Fiji (FJT)', timezone: 'Pacific/Fiji', zone: 'Pacific'}, {label: 'Beijing (CST)', timezone: 'Asia/Shanghai', zone: 'Asia'}, {label: 'Tokyo (JST)', timezone: 'Asia/Tokyo', zone: 'Asia'}, {label: 'Seoul (KST)', timezone: 'Asia/Seoul', zone: 'Asia'}, From 0572f0b9bcc80567d8a77a9edf285994beab91aa Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Fri, 31 Oct 2025 09:28:20 -0500 Subject: [PATCH 06/18] feat(date-timezone): add New Zealand and Fiji timezones to the constants Signed-off-by: Manuel Abascal --- CHANGELOG.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d19e991d1..d3069b206 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,3 @@ -# UTMStack 10.9.1 Release Notes +# UTMStack 10.9.2 Release Notes --- Dashboard Rendering with Time Filters - Resolved performance issues affecting dashboard responsiveness when applying time-based filters. \ No newline at end of file +-- Added new Pacific time zones (New Zealand and Fiji) to the Date Settings section. \ No newline at end of file From a6dd65629755df85c870d5799a2a1c2336d133c4 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Fri, 31 Oct 2025 10:00:04 -0500 Subject: [PATCH 07/18] fea: add tls connection options and setup steps for syslog integrations Signed-off-by: Manuel Abascal --- .../install-log-collector.component.ts | 2 +- .../components/log-collector.component.ts | 57 ++++++------- .../shared/components/step.component.ts | 2 +- .../app/app-module/guides/shared/constant.ts | 84 +++++++++++++++++++ .../util/replace-command-tokens.util.ts | 22 +++++ version.yml | 2 +- 6 files changed, 136 insertions(+), 33 deletions(-) create mode 100644 frontend/src/app/app-module/guides/shared/constant.ts create mode 100644 frontend/src/app/shared/util/replace-command-tokens.util.ts diff --git a/frontend/src/app/app-module/guides/shared/components/install-log-collector.component.ts b/frontend/src/app/app-module/guides/shared/components/install-log-collector.component.ts index bfe5c951f..61d1e80ed 100644 --- a/frontend/src/app/app-module/guides/shared/components/install-log-collector.component.ts +++ b/frontend/src/app/app-module/guides/shared/components/install-log-collector.component.ts @@ -4,7 +4,7 @@ import { ModalConfirmationComponent } from '../../../../shared/components/utm/util/modal-confirmation/modal-confirmation.component'; import {UtmModulesEnum} from '../../../shared/enum/utm-module.enum'; -import {generate} from "rxjs"; +import {generate} from 'rxjs'; @Component({ selector: 'app-install-log-collector', diff --git a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts index 3a6777a8b..f248e7425 100644 --- a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts +++ b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts @@ -3,6 +3,7 @@ import {ModalService} from '../../../../core/modal/modal.service'; import { ModalConfirmationComponent } from '../../../../shared/components/utm/util/modal-confirmation/modal-confirmation.component'; +import {replaceCommandTokens} from "../../../../shared/util/replace-command-tokens.util"; import {UtmModulesEnum} from '../../../shared/enum/utm-module.enum'; @Component({ @@ -31,7 +32,7 @@ import {UtmModulesEnum} from '../../../shared/enum/utm-module.enum'; {{selectedPlatform.shell}} - + `, styles: [` @@ -49,9 +50,14 @@ import {UtmModulesEnum} from '../../../shared/enum/utm-module.enum'; export class LogCollectorComponent { + @Input() agent: string; + @Input() platforms: any[]; + @Input() hideActions = false; + @Input() hideProtocols = false; @Input() protocols = [ {id: 1, name: 'TCP'}, - {id: 2, name: 'UDP'} + {id: 2, name: 'TCP/TLS'}, + {id: 3, name: 'UDP'} ]; actions = [ @@ -59,46 +65,37 @@ export class LogCollectorComponent { {id: 2, name: 'DISABLE', action: 'disable-integration'} ]; - platforms = [ - { - id: 1, name: 'WINDOWS (ARM64)', - command: 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service_arm64.exe" -ArgumentList \'ACTION\', \'AGENTNAME\', \'PORT\' -NoNewWindow -Wait\n', - shell: 'Windows Powershell terminal as “ADMINISTRATOR”' - }, - { - id: 2, name: 'WINDOWS (AMD64)', - command: 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service.exe" -ArgumentList \'ACTION\', \'AGENTNAME\', \'PORT\' -NoNewWindow -Wait\n', - shell: 'Windows Powershell terminal as “ADMINISTRATOR”' - }, - { - id: 3, - name: 'LINUX', command: 'sudo bash -c "/opt/utmstack-linux-agent/utmstack_agent_service ACTION AGENTNAME PORT"', - shell: 'Linux bash terminal' - } - ]; - - @Input() agent: string; - _selectedProtocol: any; _selectedPlatform: any; _selectedAction: any; module = UtmModulesEnum; - constructor(private modalService: ModalService) { - } + constructor(private modalService: ModalService) {} - get command() { - return this.replaceAll(this.selectedPlatform.command, { - PORT: this.selectedProtocol.name.toLowerCase(), - AGENTNAME: this.agentName(), - ACTION: this.selectedAction.action - }); + get commands() { + + const protocol = this.selectedProtocol && this.selectedProtocol.name === 'TCP/TLS' ? 'tcp' : this.selectedProtocol.name.toLowerCase(); + + const command = replaceCommandTokens(this.selectedPlatform.command, { + PROTOCOL: protocol, + AGENT_NAME: this.agent, + ACTION: this.selectedAction && this.selectedAction.action || '', + TLS: this.selectedProtocol && this.selectedProtocol.name === 'TCP/TLS' ? ' --tls' : '' + }); + + if (this.selectedProtocol && this.selectedProtocol.name === 'TCP/TLS') { + const extras = this.selectedPlatform.extraCommands ? this.selectedPlatform.extraCommands : []; + return [...extras, command]; + } + + return [command]; } get selectedPlatform() { return this._selectedPlatform; } + @Input() set selectedPlatform(platform) { this._selectedPlatform = platform; } diff --git a/frontend/src/app/app-module/guides/shared/components/step.component.ts b/frontend/src/app/app-module/guides/shared/components/step.component.ts index 416e0f1fa..038fae1a3 100644 --- a/frontend/src/app/app-module/guides/shared/components/step.component.ts +++ b/frontend/src/app/app-module/guides/shared/components/step.component.ts @@ -22,7 +22,7 @@ export class StepDirective { ` }) -export class StepComponent{ +export class StepComponent { @Input() step: Step; constructor() { diff --git a/frontend/src/app/app-module/guides/shared/constant.ts b/frontend/src/app/app-module/guides/shared/constant.ts new file mode 100644 index 000000000..5be905914 --- /dev/null +++ b/frontend/src/app/app-module/guides/shared/constant.ts @@ -0,0 +1,84 @@ +export interface Platform { + id: number; + name: string; + command: string; + shell: string; + path: string; + restart: string; +} + +export const createPlatforms = (windowsCommandAMD64: string, + windowsCommandARM64: string, + linuxCommand: string, + windowsPath?: string, + windowsRestart?: string, + linuxPath?: string, + linuxRestart?: string) => [ + { + id: 1, + name: 'WINDOWS (AMD64)', + command: windowsCommandAMD64, + shell: 'Run the following powershell script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', + path: windowsPath, + restart: windowsRestart + }, + { + id: 2, + name: 'WINDOWS (ARM64)', + command: windowsCommandARM64, + shell: 'Run the following powershell script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', + path: windowsPath, + restart: windowsRestart + }, + { + id: 3, + name: 'LINUX', + command: linuxCommand, + shell: 'Run the following bash script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', + path: linuxPath, + restart: linuxRestart + } +]; + +export const createFileBeatsPlatforms = (windowsCommand: string, + linuxCommand: string, + windowsPath?: string, + windowsRestart?: string, + linuxPath?: string, + linuxRestart?: string) => [ + { + id: 1, + name: 'WINDOWS', + command: windowsCommand, + shell: 'Run the following powershell script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', + path: windowsPath, + restart: windowsRestart + }, + { + id: 3, + name: 'LINUX', + command: linuxCommand, + shell: 'Run the following bash script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', + path: linuxPath, + restart: linuxRestart + } +]; + + +export const PLATFORMS = createPlatforms( + 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service.exe" -ArgumentList \'ACTION\',' + + ' \'AGENT_NAME\', \'PORT\' -NoNewWindow -Wait\n', + 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service_arm64.exe" -ArgumentList \'ACTION\',' + + ' \'AGENT_NAME\', \'PORT\' -NoNewWindow -Wait\n', + 'sudo bash -c "/opt/utmstack-linux-agent/utmstack_agent_service ACTION AGENT_NAME PORT"' +); + + +export const FILEBEAT_PLATFORMS = createFileBeatsPlatforms( + 'cd "C:\\Program Files\\UTMStack\\UTMStack Agent\\beats\\filebeat\\"; Start-Process "filebeat.exe" -ArgumentList "modules", "enable", \"AGENT_NAME\"', + 'cd /opt/utmstack-linux-agent/beats/filebeat/ && ./filebeat modules enable AGENT_NAME', + 'C:\\Program Files\\UTMStack\\UTMStack Agent\\beats\\filebeat\\modules.d\\', + 'Stop-Service -Name UTMStackModulesLogsCollector; Start-Sleep -Seconds 5; Start-Service -Name UTMStackModulesLogsCollector', + '/opt/utmstack-linux-agent/beats/filebeat/modules.d/', + 'sudo systemctl restart UTMStackModulesLogsCollector' +); diff --git a/frontend/src/app/shared/util/replace-command-tokens.util.ts b/frontend/src/app/shared/util/replace-command-tokens.util.ts new file mode 100644 index 000000000..b130861cc --- /dev/null +++ b/frontend/src/app/shared/util/replace-command-tokens.util.ts @@ -0,0 +1,22 @@ +/*export function replaceCommandTokens(command: string, wordsToReplace: { [key: string]: string }) { + return Object.keys(wordsToReplace) + .reduce((f, s) => f.replace(new RegExp(s, 'ig'), wordsToReplace[s]), command); +}*/ + +export function replaceCommandTokens(command: string, wordsToReplace: { [key: string]: string }) { + let cmd = command; + + Object.entries(wordsToReplace).forEach(([key, value]) => { + if (!value) { + const regex = new RegExp(`\\s*${key}\\b`, 'g'); + cmd = cmd.replace(regex, ''); + } else { + const regex = new RegExp(`${key}\\b`, 'g'); + cmd = cmd.replace(regex, value); + } + }); + + cmd = cmd.replace(/\s+/g, ' ').trim(); + + return cmd; +} diff --git a/version.yml b/version.yml index 9801dbb55..fdcb6d806 100644 --- a/version.yml +++ b/version.yml @@ -1 +1 @@ -version: 10.9.1 \ No newline at end of file +version: 10.9.2 \ No newline at end of file From 8d9be89a49d4985e863beaa789b1cb3ef413f5c0 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Fri, 31 Oct 2025 10:16:11 -0500 Subject: [PATCH 08/18] fea: add tls connection options and setup steps for syslog integrations Signed-off-by: Manuel Abascal --- .../guides/guide-syslog/guide-syslog.component.html | 3 +-- .../guides/shared/components/log-collector.component.ts | 5 +++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/frontend/src/app/app-module/guides/guide-syslog/guide-syslog.component.html b/frontend/src/app/app-module/guides/guide-syslog/guide-syslog.component.html index 413d8a646..884ce829e 100644 --- a/frontend/src/app/app-module/guides/guide-syslog/guide-syslog.component.html +++ b/frontend/src/app/app-module/guides/guide-syslog/guide-syslog.component.html @@ -31,8 +31,7 @@

- + diff --git a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts index f248e7425..6ff356492 100644 --- a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts +++ b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts @@ -5,6 +5,7 @@ import { } from '../../../../shared/components/utm/util/modal-confirmation/modal-confirmation.component'; import {replaceCommandTokens} from "../../../../shared/util/replace-command-tokens.util"; import {UtmModulesEnum} from '../../../shared/enum/utm-module.enum'; +import {PLATFORMS} from "../constant"; @Component({ selector: 'app-log-colletor', @@ -32,7 +33,7 @@ import {UtmModulesEnum} from '../../../shared/enum/utm-module.enum'; {{selectedPlatform.shell}} - + `, styles: [` @@ -51,7 +52,7 @@ import {UtmModulesEnum} from '../../../shared/enum/utm-module.enum'; export class LogCollectorComponent { @Input() agent: string; - @Input() platforms: any[]; + @Input() platforms: any[] = PLATFORMS; @Input() hideActions = false; @Input() hideProtocols = false; @Input() protocols = [ From 8ea7022ca5c136bad7fbc8f513f5a2bc1a18b181 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Fri, 31 Oct 2025 10:38:33 -0500 Subject: [PATCH 09/18] fea: add tls connection options and setup steps for syslog integrations Signed-off-by: Manuel Abascal --- .../components/log-collector.component.ts | 2 +- .../app/app-module/guides/shared/constant.ts | 154 +++++++++++------- 2 files changed, 92 insertions(+), 64 deletions(-) diff --git a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts index 6ff356492..6da867293 100644 --- a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts +++ b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts @@ -79,7 +79,7 @@ export class LogCollectorComponent { const command = replaceCommandTokens(this.selectedPlatform.command, { PROTOCOL: protocol, - AGENT_NAME: this.agent, + AGENT_NAME: this.agentName(), ACTION: this.selectedAction && this.selectedAction.action || '', TLS: this.selectedProtocol && this.selectedProtocol.name === 'TCP/TLS' ? ' --tls' : '' }); diff --git a/frontend/src/app/app-module/guides/shared/constant.ts b/frontend/src/app/app-module/guides/shared/constant.ts index 5be905914..976dbe6ed 100644 --- a/frontend/src/app/app-module/guides/shared/constant.ts +++ b/frontend/src/app/app-module/guides/shared/constant.ts @@ -1,81 +1,109 @@ +const WINDOWS_SHELL = + 'Run the following PowerShell script as “ADMINISTRATOR” on a server with the UTMStack agent installed.'; + +const LINUX_SHELL = + 'Run the following Bash script as “ADMINISTRATOR” on a server with the UTMStack agent installed.'; + export interface Platform { id: number; name: string; command: string; shell: string; - path: string; - restart: string; + path?: string; + restart?: string; + extraCommands?: string[]; } -export const createPlatforms = (windowsCommandAMD64: string, - windowsCommandARM64: string, - linuxCommand: string, - windowsPath?: string, - windowsRestart?: string, - linuxPath?: string, - linuxRestart?: string) => [ - { - id: 1, - name: 'WINDOWS (AMD64)', - command: windowsCommandAMD64, - shell: 'Run the following powershell script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', - path: windowsPath, - restart: windowsRestart - }, - { - id: 2, - name: 'WINDOWS (ARM64)', - command: windowsCommandARM64, - shell: 'Run the following powershell script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', - path: windowsPath, - restart: windowsRestart - }, - { - id: 3, - name: 'LINUX', - command: linuxCommand, - shell: 'Run the following bash script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', - path: linuxPath, - restart: linuxRestart - } -]; +function createPlatform( + id: number, + name: string, + command: string, + shell: string, + path?: string, + restart?: string, + extraCommands?: string[]): Platform { + return { id, name, command, shell, path, restart, extraCommands }; +} -export const createFileBeatsPlatforms = (windowsCommand: string, - linuxCommand: string, - windowsPath?: string, - windowsRestart?: string, - linuxPath?: string, - linuxRestart?: string) => [ - { - id: 1, - name: 'WINDOWS', - command: windowsCommand, - shell: 'Run the following powershell script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', - path: windowsPath, - restart: windowsRestart - }, - { - id: 3, - name: 'LINUX', - command: linuxCommand, - shell: 'Run the following bash script as “ADMINISTRATOR” in a Server with the UTMStack agent Installed.', - path: linuxPath, - restart: linuxRestart - } +export const createPlatforms = ( + windowsCommandAMD64: string, + windowsCommandARM64: string, + linuxCommand: string, + windowsPath?: string, + windowsRestart?: string, + linuxPath?: string, + linuxRestart?: string): Platform[] => [ + createPlatform( + 1, + 'WINDOWS (AMD64)', + windowsCommandAMD64, + WINDOWS_SHELL, + windowsPath, + windowsRestart,[ + `Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service.exe" \` + -ArgumentList 'load-tls-certs', '[YOUR_CERT_PATH]', '[YOUR_KEY_PATH]' \` + -NoNewWindow -Wait` + ] + ), + createPlatform( + 2, + 'WINDOWS (ARM64)', + windowsCommandARM64, + WINDOWS_SHELL, + windowsPath, + windowsRestart, + [ + `Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service_arm64.exe" \` + -ArgumentList 'load-tls-certs', '[YOUR_CERT_PATH]', '[YOUR_KEY_PATH]' \` + -NoNewWindow -Wait` + ] + ), + createPlatform( + 3, + 'LINUX', + linuxCommand, + LINUX_SHELL, + linuxPath, + linuxRestart, + [ + `sudo bash -c "/opt/utmstack-linux-agent/utmstack_agent_service load-tls-certs [YOUR_CERT_PATH] [YOUR_KEY_PATH]"` + ] + ) ]; +export const createFileBeatsPlatforms = ( + windowsCommand: string, + linuxCommand: string, + windowsPath?: string, + windowsRestart?: string, + linuxPath?: string, + linuxRestart?: string): Platform[] => [ + createPlatform( + 1, + 'WINDOWS', + windowsCommand, + WINDOWS_SHELL, + windowsPath, + windowsRestart + ), + createPlatform( + 3, + 'LINUX', + linuxCommand, + LINUX_SHELL, + linuxPath, + linuxRestart + ) +]; export const PLATFORMS = createPlatforms( - 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service.exe" -ArgumentList \'ACTION\',' + - ' \'AGENT_NAME\', \'PORT\' -NoNewWindow -Wait\n', - 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service_arm64.exe" -ArgumentList \'ACTION\',' + - ' \'AGENT_NAME\', \'PORT\' -NoNewWindow -Wait\n', - 'sudo bash -c "/opt/utmstack-linux-agent/utmstack_agent_service ACTION AGENT_NAME PORT"' + 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service.exe" -ArgumentList \'ACTION\', \'AGENT_NAME\', \'PROTOCOL\' TLS -NoNewWindow -Wait\n', + 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service_arm64.exe" -ArgumentList \'ACTION\', \'AGENT_NAME\', \'PROTOCOL\' TLS -NoNewWindow -Wait\n', + 'sudo bash -c "/opt/utmstack-linux-agent/utmstack_agent_service ACTION AGENT_NAME PROTOCOL TLS"' ); - export const FILEBEAT_PLATFORMS = createFileBeatsPlatforms( - 'cd "C:\\Program Files\\UTMStack\\UTMStack Agent\\beats\\filebeat\\"; Start-Process "filebeat.exe" -ArgumentList "modules", "enable", \"AGENT_NAME\"', + 'cd "C:\\Program Files\\UTMStack\\UTMStack Agent\\beats\\filebeat\\"; Start-Process "filebeat.exe" -ArgumentList "modules", "enable", "AGENT_NAME"', 'cd /opt/utmstack-linux-agent/beats/filebeat/ && ./filebeat modules enable AGENT_NAME', 'C:\\Program Files\\UTMStack\\UTMStack Agent\\beats\\filebeat\\modules.d\\', 'Stop-Service -Name UTMStackModulesLogsCollector; Start-Sleep -Seconds 5; Start-Service -Name UTMStackModulesLogsCollector', From d5c7a3ebbebf937b5314540aacd7533a96fdd8f8 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Fri, 31 Oct 2025 10:39:59 -0500 Subject: [PATCH 10/18] feat: add tls connection options and setup steps for syslog integrations Signed-off-by: Manuel Abascal --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d3069b206..066f82d1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,4 @@ # UTMStack 10.9.2 Release Notes --- Added new Pacific time zones (New Zealand and Fiji) to the Date Settings section. \ No newline at end of file +-- Added new Pacific time zones (New Zealand and Fiji) to the Date Settings section. +-- Added TLS connection options and setup steps for secure Syslog integration \ No newline at end of file From e3ca85b1ca570276301523bf86163a7e6bb2480d Mon Sep 17 00:00:00 2001 From: Yadian Llada Lopez Date: Fri, 31 Oct 2025 13:23:04 -0400 Subject: [PATCH 11/18] feat(agent): validate TLS certificates before enabling TLS for integrations --- agent/modules/configuration.go | 4 ++++ agent/modules/syslog.go | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/agent/modules/configuration.go b/agent/modules/configuration.go index ba55f856a..e551f7c84 100644 --- a/agent/modules/configuration.go +++ b/agent/modules/configuration.go @@ -84,6 +84,9 @@ func ChangeIntegrationStatus(logTyp string, proto string, isEnabled bool, tlsOpt // Handle TLS configuration if specified if len(tlsOptions) > 0 && isEnabled { if tlsOptions[0] { + if !utils.CheckIfPathExist(config.IntegrationCertPath) || !utils.CheckIfPathExist(config.IntegrationKeyPath) { + return "", fmt.Errorf("TLS certificates not found. Please load certificates first") + } // Enable TLS integration.TCP.TLSEnabled = true mod := GetModule(logTyp) @@ -249,6 +252,7 @@ func EnableTLSForIntegration(logTyp string, proto string) (string, error) { if integration.TCP.Port == "" { return "", fmt.Errorf("TCP port not configured for %s", logTyp) } + port = integration.TCP.Port integration.TCP.TLSEnabled = true diff --git a/agent/modules/syslog.go b/agent/modules/syslog.go index 4c02fd5aa..57bba9539 100644 --- a/agent/modules/syslog.go +++ b/agent/modules/syslog.go @@ -5,6 +5,7 @@ import ( "context" "crypto/tls" "errors" + "fmt" "io" "net" "os" @@ -96,6 +97,12 @@ func (m *SyslogModule) GetPort(proto string) string { func (m *SyslogModule) EnablePort(proto string, enableTLS bool) error { switch proto { case "tcp": + if enableTLS { + if !utils.CheckIfPathExist(config.IntegrationCertPath) || !utils.CheckIfPathExist(config.IntegrationKeyPath) { + return fmt.Errorf("TLS certificates not found. Please load certificates first") + } + } + // Update TLS configuration before enabling m.TCPListener.TLSEnabled = enableTLS go m.enableTCP() From 0a89a40a6b8af734731f8e7e0c80088dca621b49 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Mon, 3 Nov 2025 10:21:07 -0600 Subject: [PATCH 12/18] feat: add tls connection options and setup steps for syslog integrations Signed-off-by: Manuel Abascal --- .../components/log-collector.component.ts | 22 ++++------- .../app/app-module/guides/shared/constant.ts | 4 +- .../util/replace-command-tokens.util.ts | 39 ++++++++++++------- 3 files changed, 35 insertions(+), 30 deletions(-) diff --git a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts index 6da867293..745885ba2 100644 --- a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts +++ b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts @@ -3,9 +3,9 @@ import {ModalService} from '../../../../core/modal/modal.service'; import { ModalConfirmationComponent } from '../../../../shared/components/utm/util/modal-confirmation/modal-confirmation.component'; -import {replaceCommandTokens} from "../../../../shared/util/replace-command-tokens.util"; +import {replaceCommandTokens} from '../../../../shared/util/replace-command-tokens.util'; import {UtmModulesEnum} from '../../../shared/enum/utm-module.enum'; -import {PLATFORMS} from "../constant"; +import {PLATFORMS} from '../constant'; @Component({ selector: 'app-log-colletor', @@ -78,13 +78,15 @@ export class LogCollectorComponent { const protocol = this.selectedProtocol && this.selectedProtocol.name === 'TCP/TLS' ? 'tcp' : this.selectedProtocol.name.toLowerCase(); const command = replaceCommandTokens(this.selectedPlatform.command, { - PROTOCOL: protocol, - AGENT_NAME: this.agentName(), ACTION: this.selectedAction && this.selectedAction.action || '', - TLS: this.selectedProtocol && this.selectedProtocol.name === 'TCP/TLS' ? ' --tls' : '' + AGENT_NAME: this.agentName(), + PROTOCOL: protocol, + TLS: this.selectedProtocol && this.selectedProtocol.name === 'TCP/TLS' && + this.selectedAction.name === 'ENABLE' ? `--tls` : '' }); - if (this.selectedProtocol && this.selectedProtocol.name === 'TCP/TLS') { + if (this.selectedProtocol && this.selectedProtocol.name === 'TCP/TLS' && + this.selectedAction.name === 'ENABLE') { const extras = this.selectedPlatform.extraCommands ? this.selectedPlatform.extraCommands : []; return [...extras, command]; } @@ -179,14 +181,6 @@ export class LogCollectorComponent { } } - replaceAll(command, wordsToReplace) { - return Object.keys(wordsToReplace).reduce( - (f, s, i) => - `${f}`.replace(new RegExp(s, 'ig'), wordsToReplace[s]), - command - ); - } - onChangeAction(action: any) { if (this.selectedPlatform && this.selectedProtocol && action.name === 'DISABLE') { this.openModal(); diff --git a/frontend/src/app/app-module/guides/shared/constant.ts b/frontend/src/app/app-module/guides/shared/constant.ts index 976dbe6ed..92ea97d80 100644 --- a/frontend/src/app/app-module/guides/shared/constant.ts +++ b/frontend/src/app/app-module/guides/shared/constant.ts @@ -97,8 +97,8 @@ export const createFileBeatsPlatforms = ( ]; export const PLATFORMS = createPlatforms( - 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service.exe" -ArgumentList \'ACTION\', \'AGENT_NAME\', \'PROTOCOL\' TLS -NoNewWindow -Wait\n', - 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service_arm64.exe" -ArgumentList \'ACTION\', \'AGENT_NAME\', \'PROTOCOL\' TLS -NoNewWindow -Wait\n', + 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service.exe" -ArgumentList \'ACTION\', \'AGENT_NAME\', \'PROTOCOL\', \'TLS\' -NoNewWindow -Wait\n', + 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service_arm64.exe" -ArgumentList \'ACTION\', \'AGENT_NAME\', \'PROTOCOL\, \'TLS\' -NoNewWindow -Wait\n', 'sudo bash -c "/opt/utmstack-linux-agent/utmstack_agent_service ACTION AGENT_NAME PROTOCOL TLS"' ); diff --git a/frontend/src/app/shared/util/replace-command-tokens.util.ts b/frontend/src/app/shared/util/replace-command-tokens.util.ts index b130861cc..7dbddc16a 100644 --- a/frontend/src/app/shared/util/replace-command-tokens.util.ts +++ b/frontend/src/app/shared/util/replace-command-tokens.util.ts @@ -1,20 +1,31 @@ -/*export function replaceCommandTokens(command: string, wordsToReplace: { [key: string]: string }) { - return Object.keys(wordsToReplace) - .reduce((f, s) => f.replace(new RegExp(s, 'ig'), wordsToReplace[s]), command); -}*/ - -export function replaceCommandTokens(command: string, wordsToReplace: { [key: string]: string }) { +export function replaceCommandTokens(command: string, wordsToReplace: { [key: string]: string }): string { let cmd = command; - Object.entries(wordsToReplace).forEach(([key, value]) => { - if (!value) { - const regex = new RegExp(`\\s*${key}\\b`, 'g'); - cmd = cmd.replace(regex, ''); - } else { - const regex = new RegExp(`${key}\\b`, 'g'); - cmd = cmd.replace(regex, value); + if (cmd.includes('-ArgumentList')) { + + const args = Object.values(wordsToReplace) + .filter(v => v && v.trim().length > 0) + .map(v => `'${v.trim()}'`) + .join(', '); + + cmd = cmd.replace( + /-ArgumentList\s+(['"].*?['"])(?=\s+-|$)/, + `-ArgumentList ${args}` + ); + } else { + const match = cmd.match(/"(.*)"/); + if (match) { + const original = match[1]; + const parts = original.split(/\s+/); + const fixedCommand = parts[0]; + const args = Object.entries(wordsToReplace) + .filter(([_, v]) => v && v.trim().length > 0) + .map(([_, v]) => v.trim()) + .join(' '); + + cmd = cmd.replace(/"(.*)"/, `"${fixedCommand} ${args}"`); } - }); + } cmd = cmd.replace(/\s+/g, ' ').trim(); From 08ff88dab7b91e27a8dfa57463b9c53602104330 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Mon, 3 Nov 2025 11:25:56 -0600 Subject: [PATCH 13/18] feat: improve asset source sorting and display in assets view Signed-off-by: Manuel Abascal --- CHANGELOG.md | 3 +- .../assets-view/assets-view.component.html | 4 +- .../assets-view/assets-view.component.ts | 87 ++++++++++++++----- .../shared/types/net-scan.type.ts | 2 + 4 files changed, 70 insertions(+), 26 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 066f82d1c..48e219e53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,5 @@ # UTMStack 10.9.2 Release Notes -- Added new Pacific time zones (New Zealand and Fiji) to the Date Settings section. --- Added TLS connection options and setup steps for secure Syslog integration \ No newline at end of file +-- Added TLS connection options and setup steps for secure Syslog integration +-- Improved sorting of asset sources in tables, ensuring consistent and predictable order for names, IPs, and combined entries. \ No newline at end of file diff --git a/frontend/src/app/assets-discover/assets-view/assets-view.component.html b/frontend/src/app/assets-discover/assets-view/assets-view.component.html index 7f69ced00..176582cb0 100644 --- a/frontend/src/app/assets-discover/assets-view/assets-view.component.html +++ b/frontend/src/app/assets-discover/assets-view/assets-view.component.html @@ -98,7 +98,7 @@
Source @@ -143,7 +143,7 @@
- {{getAssetSource(asset)}} + {{ asset.displayName }}
{ + this.utmNetScanService.query(this.requestParam) + .subscribe(response => { this.totalItems = Number(response.headers.get('X-Total-Count')); - this.assets = response.body; + const assets = response.body || []; + + this.assets = assets.map(asset => { + const displayName = + asset.assetName && asset.assetIp + ? `${asset.assetName} (${asset.assetIp})` + : asset.assetName + ? asset.assetName + : asset.assetIp + ? asset.assetIp + : 'Unknown source'; + + const sortKey = (asset.assetName || '') + (asset.assetIp || ''); + + return { ...asset, displayName, sortKey }; + }); + + console.log(this.assets); + this.loading = false; }); } @@ -154,8 +169,42 @@ export class AssetsViewComponent implements OnInit, OnDestroy { } onSortBy($event: SortEvent) { - this.requestParam.sort = $event.column + ',' + $event.direction; - this.getAssets(); + if ($event.column === 'displayName') { + this.sortAssets($event.direction); + } else { + this.requestParam.sort = $event.column + ',' + $event.direction; + this.getAssets(); + } + } + + sortAssets(direction: SortDirection) { + this.assets.sort((a, b) => { + + if (a.displayName === 'Unknown source') { return 1; } + if (b.displayName === 'Unknown source') { return -1; } + + const aVal = a.sortKey!; + const bVal = b.sortKey!; + + const ipRegex = /^(\d{1,3}\.){3}\d{1,3}$/; + const aIsIP = ipRegex.test(aVal); + const bIsIP = ipRegex.test(bVal); + + if (aIsIP && bIsIP) { + const aOctets = aVal.split('.').map(Number); + const bOctets = bVal.split('.').map(Number); + for (let i = 0; i < 4; i++) { + if (aOctets[i] !== bOctets[i]) { return direction === 'asc' ? aOctets[i] - bOctets[i] : bOctets[i] - aOctets[i]; } + } + return 0; + } + + if (aIsIP) { return direction === 'asc' ? -1 : 1; } + if (bIsIP) { return direction === 'asc' ? 1 : -1; } + + const cmp = aVal.localeCompare(bVal, undefined, { numeric: true, sensitivity: 'base' }); + return direction === 'asc' ? cmp : -cmp; + }); } toggleCheck() { @@ -279,19 +328,6 @@ export class AssetsViewComponent implements OnInit, OnDestroy { }); } - - getAssetSource(asset: NetScanType) { - if (asset.assetName && asset.assetIp) { - return asset.assetName + ' (' + asset.assetIp + ')'; - } else if (asset.assetName) { - return asset.assetName; - } else if (asset.assetIp) { - return asset.assetIp; - } else { - return 'Unknown source'; - } - } - navigateToDataManagement(ip: string) { const queryParams = {alertType: 'ALERT'}; queryParams[ALERT_SENSOR_FIELD] = ElasticOperatorsEnum.IS + ChartValueSeparator.BUCKET_SEPARATOR + ip; @@ -374,7 +410,12 @@ export class AssetsViewComponent implements OnInit, OnDestroy { if (!this.interval) { this.interval = setInterval(() => { this.getAssets(); - }, 10000); + }, 60000); } } + + ngOnDestroy(): void { + this.stopInterval(true); + this.assetFiltersBehavior.$assetFilter.next(null); + } } diff --git a/frontend/src/app/assets-discover/shared/types/net-scan.type.ts b/frontend/src/app/assets-discover/shared/types/net-scan.type.ts index 0d10ede9b..3d2b67cb0 100644 --- a/frontend/src/app/assets-discover/shared/types/net-scan.type.ts +++ b/frontend/src/app/assets-discover/shared/types/net-scan.type.ts @@ -35,4 +35,6 @@ export class NetScanType { assetOsPlatform?: string; assetOsMinorVersion?: string; assetOsMajorVersion?: string; + displayName?: string; + sortKey?: string; } From 9c9d5d9b74e075c2488b42625853ab582e70eb25 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Wed, 5 Nov 2025 07:35:41 -0600 Subject: [PATCH 14/18] feat(agent): add TLS support for syslog command generation and update argument handling Signed-off-by: Manuel Abascal --- .../guides/shared/components/log-collector.component.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts index 745885ba2..5829e4b09 100644 --- a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts +++ b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts @@ -31,6 +31,12 @@ import {PLATFORMS} from '../constant'; class="flex-item">
+
+ After the TLS certificates have been successfully loaded into the system, + it is not necessary to repeat the certificate loading process when enabling + additional integrations that use TLS. The system will automatically apply the + previously configured certificates to ensure secure communication. +
{{selectedPlatform.shell}} From 2432a3565d37be1ffdc0fc70c39b6c02106f6041 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Wed, 5 Nov 2025 07:39:18 -0600 Subject: [PATCH 15/18] feat(agent): add TLS support for syslog command generation and update argument handling Signed-off-by: Manuel Abascal --- .../guides/shared/components/log-collector.component.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts index 5829e4b09..5c0cc054d 100644 --- a/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts +++ b/frontend/src/app/app-module/guides/shared/components/log-collector.component.ts @@ -31,7 +31,8 @@ import {PLATFORMS} from '../constant'; class="flex-item"> -
+
After the TLS certificates have been successfully loaded into the system, it is not necessary to repeat the certificate loading process when enabling additional integrations that use TLS. The system will automatically apply the From 23adb5fd543ea1e363a3329d36220c5f976fac52 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Thu, 6 Nov 2025 10:53:27 -0600 Subject: [PATCH 16/18] feat(agent): refactor PowerShell command generation for TLS certificate loading Signed-off-by: Manuel Abascal --- .../src/app/app-module/guides/shared/constant.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/frontend/src/app/app-module/guides/shared/constant.ts b/frontend/src/app/app-module/guides/shared/constant.ts index 92ea97d80..468dd47fc 100644 --- a/frontend/src/app/app-module/guides/shared/constant.ts +++ b/frontend/src/app/app-module/guides/shared/constant.ts @@ -40,9 +40,9 @@ export const createPlatforms = ( WINDOWS_SHELL, windowsPath, windowsRestart,[ - `Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service.exe" \` - -ArgumentList 'load-tls-certs', '[YOUR_CERT_PATH]', '[YOUR_KEY_PATH]' \` - -NoNewWindow -Wait` + 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service.exe" ' + + '-ArgumentList \'load-tls-certs\', \'[YOUR_CERT_PATH]\', \'[YOUR_KEY_PATH]\' ' + + '-NoNewWindow -Wait' ] ), createPlatform( @@ -53,9 +53,9 @@ export const createPlatforms = ( windowsPath, windowsRestart, [ - `Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service_arm64.exe" \` - -ArgumentList 'load-tls-certs', '[YOUR_CERT_PATH]', '[YOUR_KEY_PATH]' \` - -NoNewWindow -Wait` + 'Start-Process "C:\\Program Files\\UTMStack\\UTMStack Agent\\utmstack_agent_service_arm64.exe" ' + + '-ArgumentList \'load-tls-certs\', \'[YOUR_CERT_PATH]\', \'[YOUR_KEY_PATH]\' ' + + '-NoNewWindow -Wait' ] ), createPlatform( From 6862ada66a7b7aeec1c2d8db0596031b4bfdfb43 Mon Sep 17 00:00:00 2001 From: Manuel Abascal Date: Sun, 9 Nov 2025 20:36:41 -0600 Subject: [PATCH 17/18] feat: add Liquibase changelogs for pfSense and SonicWall filter updates --- CHANGELOG.md | 3 +- .../20251107001_update_filter_pfsense.xml | 268 ++++++++++ .../20251107002_update_filter_sonicwall.xml | 482 ++++++++++++++++++ .../resources/config/liquibase/master.xml | 4 + 4 files changed, 756 insertions(+), 1 deletion(-) create mode 100644 backend/src/main/resources/config/liquibase/changelog/20251107001_update_filter_pfsense.xml create mode 100644 backend/src/main/resources/config/liquibase/changelog/20251107002_update_filter_sonicwall.xml diff --git a/CHANGELOG.md b/CHANGELOG.md index 48e219e53..4e33dff40 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,4 +2,5 @@ -- Added new Pacific time zones (New Zealand and Fiji) to the Date Settings section. -- Added TLS connection options and setup steps for secure Syslog integration --- Improved sorting of asset sources in tables, ensuring consistent and predictable order for names, IPs, and combined entries. \ No newline at end of file +-- Improved sorting of asset sources in tables, ensuring consistent and predictable order for names, IPs, and combined entries. +–- Improved correlation rule handling for pfSense and SonicWall data sources to enhance detection accuracy and event normalization. \ No newline at end of file diff --git a/backend/src/main/resources/config/liquibase/changelog/20251107001_update_filter_pfsense.xml b/backend/src/main/resources/config/liquibase/changelog/20251107001_update_filter_pfsense.xml new file mode 100644 index 000000000..5e9028910 --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20251107001_update_filter_pfsense.xml @@ -0,0 +1,268 @@ + + + + + + + "message" + terminator => "" + } + + #Looking for datasource generated by an agent and parse original message + if [message] =~ /\[utm_stack_agent_ds=(.+)\]-(.+)/ { + grok { + match => { + "message" => "\[utm_stack_agent_ds=%{DATA:dataSource}\]-%{GREEDYDATA:original_log_message}" + } + } + } + if [original_log_message] { + mutate { + update => { "message" => "%{[original_log_message]}" + } + } + } + + if ![dataType] { +# The log destination is already identified by the agent so, don''t need an entry point +#......................................................................# +#Generating dataType field required by CurrelationRulesEngine + mutate { + add_field => { "dataType" => "firewall-pfsense" } + } + + #......................................................................# + #Using grok to parse header of the message + grok { + match => { + "message" => [ + # Format with syslog header: version timestamp host program[pid]: data + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program}\[%{NUMBER:pid}\]: %{GREEDYDATA:msg_all}", + # Format with syslog header: version timestamp host program pid - - data + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program} %{NUMBER:pid} - - %{GREEDYDATA:msg_all}", + # Format with syslog header: version timestamp host program pid: data + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program} %{NUMBER:pid}: %{GREEDYDATA:msg_all}", + # Format with syslog header: version timestamp host program: data (no pid) + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{DATA:program}: %{GREEDYDATA:msg_all}", + # Original pattern (legacy support): optional number at start + timestamp + "(%{INT:not_defined})?(\s)?(<%{NUMBER:priority}>)?(%{INT:syslog_version})? %{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{GREEDYDATA:msg_all}", + # Fallback: just timestamp and host for logs without syslog headers + "%{TIMESTAMP_ISO8601:timestamp} %{IPORHOST:syslog_host} %{GREEDYDATA:msg_all}", + # Direct CSV payload without any header + "%{IPORHOST:syslog_host} %{DATA:program}.*? %{GREEDYDATA:msg_all}" + ] + } + tag_on_failure => ["_grok_header_fail"] + } + + #......................................................................# + #Checking that the msg_all field exists + if [msg_all] { + if [message] =~ / filterlog(.+),(match|\w+),(block|pass),(in|out),(4|6)/ { + grok { + match => { + "msg_all" => "%{WORD:event_type}(\s)?(\[)?(%{NUMBER:pid})?(\])?( - -|:) %{GREEDYDATA:csv_msg}" + } + } + #.....................................................................# + #Check if csv_msg exists and parsing it + if [csv_msg] { + #Changing the empty fields by X0X and then eliminating them + mutate { + gsub => [ + "csv_msg", ",,", ",X0X," + ] + } + #The gsub is repeated, because the first time it leaves some intermediate fields empty + mutate { + gsub => [ + "csv_msg", ",,", ",X0X," + ] + } + } + } else if [message] =~ / [a-z-_\.]+( \d+)? - - / { + grok { + match => { + "msg_all" => "%{DATA:event_type}( %{NUMBER:pid})? - - (- )?%{GREEDYDATA:msg}" + } + } + } else if [message] =~ /\/(.+?)( \d+)? - - \((.+?)\) [A-Z]+ \((.+?)\)/ { + grok { + match => { + "msg_all" => "%{PATH:process}( %{NUMBER:pid})? - - \(%{DATA:user}\) %{WORD:command_action} \(%{PATH:process_path}\)" + } + } + } + } + + #......................................................................# + # Rename other fields + mutate { + rename => { + "event_type" => "[logx][pfsense][event_type]" + "msg" => "[logx][pfsense][msg]" + "pid" => "[logx][pfsense][pid]" + "priority" => "[logx][pfsense][priority]" + "message" => "[logx][pfsense][message]" + } + } + + #.....................................................................# + #Generating dataSource field required by Correlation Engine + if [syslog_host] { + mutate { + rename => { "syslog_host" => "[logx][pfsense][syslog_host]" } + update => { "dataSource" => "%{[logx][pfsense][syslog_host]}" } + } + } + if (![dataSource]){ + mutate { + add_field => { "dataSource" => "%{host}" } + } + } + + if [logx][pfsense][message] and [logx][pfsense][message] =~ /^ { + "[logx][pfsense][message]" => [ + "<%{NUMBER:priority}>%{INT:syslog_version} %{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}(?: %{NUMBER:pid})? - - %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}\[%{NUMBER:pid}\]: %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program} %{NUMBER:pid} - - %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program} %{NUMBER:pid}: %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}: %{GREEDYDATA:payload}", + + "<%{NUMBER:priority}>(?:%{INT:syslog_version} )?%{TIMESTAMP_ISO8601:deviceTime} %{DATA:syslog_host} %{DATA:program}.*? %{GREEDYDATA:payload}" + ] + } + tag_on_failure => ["_grok2_fail"] + } + if !("_grok2_fail" in [tags]) and [payload] { + mutate { + replace => { "[logx][pfsense][message]" => "%{[payload]}" } + } + } + } + #......................................................................# + #Remove unnecessary fields + mutate { + remove_field => [ + "@version","timestamp","payload","deviceTime","path","type","syslog_version","syslog_host","not_defined", + "pid","priority","program","event","csv_msg","csv_field","msg_all","original_log_message","tags","headers" + ] + } + } + + if [logx][pfsense][message] { + + #......................................................................# + #IPv4 branches + if [logx][pfsense][message] =~ /,4,/ { + + #......................................................................# + #IPv4 ICMP: map ICMP-specific tail + if [logx][pfsense][message] =~ /,icmp,/ { + csv { + source => "[logx][pfsense][message]" + target => "[logx][pfsense]" + separator => "," + skip_empty_columns => true + columns => [ + "rule_number","sub_rule_number","anchor","tracker","interface","reason", + "action","direction","ip_version","ipv4_tos","ipv4_ecn","ipv4_ttl", + "ipv4_id","ipv4_offset","ipv4_flags","ipv4_protocol_id","proto","ip_length", + "src_ip","dest_ip", + "icmp_type","icmp_data1","icmp_data2","icmp_data3","icmp_data4","icmp_data5" + ] + } + + #......................................................................# + #IPv4 TCP/UDP: share list; TCP fills extra fields, UDP leaves them nil + } else { + csv { + source => "[logx][pfsense][message]" + target => "[logx][pfsense]" + separator => "," + skip_empty_columns => true + columns => [ + "rule_number","sub_rule_number","anchor","tracker","interface","reason", + "action","direction","ip_version","ipv4_tos","ipv4_ecn","ipv4_ttl", + "ipv4_id","ipv4_offset","ipv4_flags","ipv4_protocol_id","proto","ip_length", + "src_ip","dest_ip", + "src_port","dest_port","data_length", + "tcp_flags","sequence_number","ack_number","tcp_window","urg","tcp_options" + ] + } + } + } + + #......................................................................# + #IPv6 branches + else if [logx][pfsense][message] =~ /,6,/ { + + #......................................................................# + #IPv6 ICMP + if [logx][pfsense][message] =~ /,icmp,/ { + csv { + source => "[logx][pfsense][message]" + target => "[logx][pfsense]" + separator => "," + skip_empty_columns => true + columns => [ + "rule_number","sub_rule_number","anchor","tracker","interface","reason", + "action","direction","ip_version","ipv6_class","ipv6_flow_label","ipv6_hop_limit", + "proto","ipv6_protocol_id","ip_length","src_ip","dest_ip", + "icmp_type","icmp_data1","icmp_data2","icmp_data3","icmp_data4","icmp_data5" + ] + } + + #......................................................................# + # IPv6 TCP/UDP + } else { + csv { + source => "[logx][pfsense][message]" + target => "[logx][pfsense]" + separator => "," + skip_empty_columns => true + columns => [ + "rule_number","sub_rule_number","anchor","tracker","interface","reason", + "action","direction","ip_version","ipv6_class","ipv6_flow_label","ipv6_hop_limit", + "proto","ipv6_protocol_id","ip_length","src_ip","dest_ip", + "src_port","dest_port","data_length", + "tcp_flags","sequence_number","ack_number","tcp_window","urg","tcp_options" + ] + } + } + } + + #......................................................................# + #Generating action field for established connections + if [logx][pfsense][action] == "pass" { + mutate { add_field => { "[logx][utm][action]" => "Success" } } + } + } + mutate { + remove_field => "[logx][pfsense][message]" + } +}' + WHERE id=1522; + ]]> + + + \ No newline at end of file diff --git a/backend/src/main/resources/config/liquibase/changelog/20251107002_update_filter_sonicwall.xml b/backend/src/main/resources/config/liquibase/changelog/20251107002_update_filter_sonicwall.xml new file mode 100644 index 000000000..510f3e154 --- /dev/null +++ b/backend/src/main/resources/config/liquibase/changelog/20251107002_update_filter_sonicwall.xml @@ -0,0 +1,482 @@ + + + + + + + "message" + terminator => "" + } + + #Looking for datasource generated by an agent and parse original message + if [message]=~/\[utm_stack_agent_ds=(.+)\]-(.+)/ { + grok { + match => { + "message" => [ "\[utm_stack_agent_ds=%{DATA:dataSource}\]-%{GREEDYDATA:original_log_message}" ] + } + } + } + if [original_log_message] { + mutate { + update => { "message" => "%{[original_log_message]}" } + } + } + + if ![dataType] { + #First,search to define the entry point contain id= and sn= and fw= and time= or CEF + if (([message] and [message] =~/\bid=\b(.+)\bsn=\b/ and [message] =~/\bfw=\b/ and [message] =~/\btime=/) + or ("CEF:" in [message] and [message] =~/\|(\w+)?(\s)?SonicWall(\s)?(\w+)?\|/)){ + #......................................................................# + #Generating dataSource field required by CorrelationRulesEngine + #Checks if exists, if not evaluate to the host variable + if (![dataSource]){ + mutate { + add_field => { "dataSource" => "%{host}" } + } + } + #......................................................................# + #Generating dataType field required by CorrelationRulesEngine + mutate { + add_field => { "dataType" => "firewall-sonicwall" } + } + #Checking if the log is in syslog format + if ([message] =~/\bid=\b(.+)\bsn=\b/ and [message] =~/\bfw=\b/ and [message] =~/\btime=/){ + #......................................................................# + #Using grok to parse header of the message + grok { + match => { + "message" => [ + "(%{INT:not_defined})?(\s)?(<%{NUMBER:priority}>)?(%{INT:syslog_version})?(%{DATA:syslog_date_host})?id=%{WORD:id} %{GREEDYDATA:msg_all}" + ] + } + } + } + #Remove unnecessary spaces from the msg_all field + if ([msg_all]){ + mutate { + strip => ["msg_all"] + } + } + #Checking if the log is in CEF format + if ("CEF:" in [message] and [message] =~/\|(\w+)?(\s)?SonicWall(\s)?(\w+)?\|/){ + #......................................................................# + #Using grok to parse header of the message + grok { + match => { + "message" => [ + "(%{INT:not_defined})?(\s)?(<%{NUMBER:priority}>)?(%{INT:syslog_version})?(\s)?%{GREEDYDATA:syslog_date_host} CEF:(\s)?%{INT:cef_version}%{GREEDYDATA:cef_msg_all}" + ] + } + } + #......................................................................# + #Checking that the cef_msg_all field exists and using grok to parse cef_msg_all + if ([cef_msg_all]){ + grok { + match => { + "cef_msg_all" => [ + "\|%{DATA:dvc_vendor}\|%{DATA:dvc_product}\|%{DATA:dvc_version}\|%{INT:event_id}\|%{DATA:event_name}\|%{INT:severity}\|%{GREEDYDATA:msg_all}" + ] + } + } + } + } + #......................................................................# + #Checking that the msg_all field exists + if ([msg_all]){ + #......................................................................# + #Using grok to parse common msg_all field components in syslog format and CEF format + grok { match => { "msg_all" => [" app=%{NUMBER:num_app_id} %{WORD:}="," app=%{GREEDYDATA:num_app_id}"] } } + grok { match => { "msg_all" => [" appcat=%{DATA:app_cat} %{WORD:}="," appcat=%{GREEDYDATA:app_cat}"] } } + grok { match => { "msg_all" => [" appid=%{DATA:appid} %{WORD:}="," appid=%{GREEDYDATA:appid}"] } } + grok { match => { "msg_all" => [" arg=%{DATA:arg} %{WORD:}="," arg=%{GREEDYDATA:arg}"] } } + grok { match => { "msg_all" => [" bcastRx=%{DATA:bcast_pkt_received} %{WORD:}="," bcastRx=%{GREEDYDATA:bcast_pkt_received}"] } } + grok { match => { "msg_all" => [" bcastTx=%{DATA:bcast_pkt_transm} %{WORD:}="," bcastTx=%{GREEDYDATA:bcast_pkt_transm}"] } } + grok { match => { "msg_all" => [" bid=%{NUMBER:blade_id} %{WORD:}="," bid=%{GREEDYDATA:blade_id}"] } } + grok { match => { "msg_all" => [" bytesRx=%{DATA:bytes_rcvd} %{WORD:}="," bytesRx=%{GREEDYDATA:bytes_rcvd}"] } } + grok { match => { "msg_all" => [" bytesTx=%{DATA:bytes_sent} %{WORD:}="," bytesTx=%{GREEDYDATA:bytes_sent}"] } } + grok { match => { "msg_all" => [" [Cc]ategory=(\")?%{DATA:cfs_category}(\")? %{WORD:}="," [Cc]ategory=%{GREEDYDATA:cfs_category}"] } } + grok { match => { "msg_all" => [" dpi=%{NUMBER:deep_packet_inspection} %{WORD:}="," dpi=%{GREEDYDATA:deep_packet_inspection}"] } } + grok { match => { "msg_all" => [" dst=%{DATA:dst} %{WORD:}="," dst=%{GREEDYDATA:dst}"] } } + grok { match => { "msg_all" => [" dstname=%{DATA:request} %{WORD:}="," dstname=%{GREEDYDATA:request}"] } } + grok { match => { "msg_all" => [" gcat=%{DATA:group_cat} %{WORD:}="," gcat=%{GREEDYDATA:group_cat}"] } } + grok { match => { "msg_all" => [" goodRxBytes=%{DATA:good_bytes_received} %{WORD:}="," goodRxBytes=%{GREEDYDATA:good_bytes_received}"] } } + grok { match => { "msg_all" => [" goodTxBytes=%{DATA:good_bytes_transm} %{WORD:}="," goodTxBytes=%{GREEDYDATA:good_bytes_transm}"] } } + grok { match => { "msg_all" => [" if=%{DATA:interface_statistics_reported} %{WORD:}="," if=%{GREEDYDATA:interface_statistics_reported}"] } } + grok { match => { "msg_all" => [" ipscat=(\")?%{DATA:ipscat}(\")? %{WORD:}="," ipscat=%{GREEDYDATA:ipscat}"] } } + grok { match => { "msg_all" => [" ipspri=%{NUMBER:ipspri} %{WORD:}="," ipspri=%{GREEDYDATA:ipspri}"] } } + grok { match => { "msg_all" => [" msg=(\")?%{DATA:msg}(\")? %{WORD:}="," msg=%{GREEDYDATA:msg}"] } } + grok { match => { "msg_all" => [" mgmtip=%{DATA:mgmt_source_ip} %{WORD:}="," mgmtip=%{GREEDYDATA:mgmt_source_ip}"] } } + grok { match => { "msg_all" => [" proto=%{DATA:protocol} %{WORD:}="," proto=%{GREEDYDATA:protocol}"] } } + grok { match => { "msg_all" => [" radio=%{DATA:radio} %{WORD:}="," radio=%{GREEDYDATA:radio}"] } } + grok { match => { "msg_all" => [" referer=(\")?%{DATA:referer}(\")? %{WORD:}="," referer=%{GREEDYDATA:referer}"] } } + grok { match => { "msg_all" => [" sid=%{NUMBER:sid} %{WORD:}="," sid=%{GREEDYDATA:sid}"] } } + grok { match => { "msg_all" => [" spycat=%{DATA:anti_spyware_cat} %{WORD:}="," spycat=%{GREEDYDATA:anti_spyware_cat}"] } } + grok { match => { "msg_all" => [" spypri=%{DATA:anti_spyware_pri} %{WORD:}="," spypri=%{GREEDYDATA:anti_spyware_pri}"] } } + grok { match => { "msg_all" => [" src=%{DATA:src} %{WORD:}="," src=%{GREEDYDATA:src}"] } } + grok { match => { "msg_all" => [" station=%{DATA:station} %{WORD:}="," station=%{GREEDYDATA:station}"] } } + grok { match => { "msg_all" => [" ucastRx=%{DATA:ucast_pkt_received} %{WORD:}="," ucastRx=%{GREEDYDATA:ucast_pkt_received}"] } } + grok { match => { "msg_all" => [" ucastTx=%{DATA:ucast_pkt_transm} %{WORD:}="," ucastTx=%{GREEDYDATA:ucast_pkt_transm}"] } } + + #......................................................................# + #Using grok to parse msg_all components from syslog format only + grok { match => { "msg_all" => [" af_polid=%{NUMBER:af_policy_id} %{WORD:}="," af_polid=%{GREEDYDATA:af_policy_id}"] } } + grok { match => { "msg_all" => [" af_policy=%{DATA:af_policy} %{WORD:}="," af_policy=%{GREEDYDATA:af_policy}"] } } + grok { match => { "msg_all" => [" af_type=%{DATA:af_policy_type} %{WORD:}="," af_type=%{GREEDYDATA:af_policy_type}"] } } + grok { match => { "msg_all" => [" af_service=%{WORD:af_policy_service} %{WORD:}="," af_service=%{GREEDYDATA:af_policy_service}"] } } + grok { match => { "msg_all" => [" af_action=%{DATA:af_policy_action} %{WORD:}="," af_action=%{GREEDYDATA:af_policy_action}"] } } + grok { match => { "msg_all" => [" af_object=%{DATA:af_policy_object} %{WORD:}="," af_object=%{GREEDYDATA:af_policy_object}"] } } + grok { match => { "msg_all" => [" ai=%{NUMBER:active_interface} %{WORD:}="," ai=%{GREEDYDATA:active_interface}"] } } + grok { match => { "msg_all" => [" appName=%{DATA:app_name} %{WORD:}="," appName=%{GREEDYDATA:app_name}"] } } + grok { match => { "msg_all" => [" c=%{NUMBER:msg_category} %{WORD:}="," c=%{GREEDYDATA:msg_category}"] } } + grok { match => { "msg_all" => [" catid=%{DATA:rule_category} %{WORD:}="," catid=%{GREEDYDATA:rule_category}"] } } + grok { match => { "msg_all" => [" cdur=%{NUMBER:conn_duration} %{WORD:}="," cdur=%{GREEDYDATA:conn_duration}"] } } + grok { match => { "msg_all" => [" change=%{DATA:change} %{WORD:}="," change=%{GREEDYDATA:change}"] } } + grok { match => { "msg_all" => [" code=%{NUMBER:code} %{WORD:}="," code=%{GREEDYDATA:code}"] } } + grok { match => { "msg_all" => [" conns=%{NUMBER:conns} %{WORD:}="," conns=%{GREEDYDATA:conns}"] } } + grok { match => { "msg_all" => [" contentObject=%{DATA:contentObject} %{WORD:}="," contentObject=%{GREEDYDATA:contentObject}"] } } + grok { match => { "msg_all" => [" icmpCode=%{DATA:icmpCode} %{WORD:}="," icmpCode=%{GREEDYDATA:icmpCode}"] } } + grok { match => { "msg_all" => [" dstMac=%{DATA:dstMac} %{WORD:}="," dstMac=%{GREEDYDATA:dstMac}"] } } + grok { match => { "msg_all" => [" dstV6=%{DATA:dstV6} %{WORD:}="," dstV6=%{GREEDYDATA:dstV6}"] } } + grok { match => { "msg_all" => [" dstZone=%{DATA:dstZone} %{WORD:}="," dstZone=%{GREEDYDATA:dstZone}"] } } + grok { match => { "msg_all" => [" dur=%{NUMBER:session_dur} %{WORD:}="," dur=%{GREEDYDATA:session_dur}"] } } + grok { match => { "msg_all" => [" dyn=%{DATA:fw_status_report} %{WORD:}="," dyn=%{GREEDYDATA:fw_status_report}"] } } + grok { match => { "msg_all" => [" f=%{NUMBER:flow_type} %{WORD:}="," f=%{GREEDYDATA:flow_type}"] } } + grok { match => { "msg_all" => [" fileid=(\")?%{DATA:fileid}(\")? %{WORD:}="," fileid=%{GREEDYDATA:fileid}"] } } + grok { match => { "msg_all" => [" filetxstatus=%{DATA:filetxstatus} %{WORD:}="," filetxstatus=%{GREEDYDATA:filetxstatus}"] } } + grok { match => { "msg_all" => [" fw=%{DATA:fw_wan} %{WORD:}="," fw=%{GREEDYDATA:fw_wan}"] } } + grok { match => { "msg_all" => [" fw_action=(\")?%{DATA:fw_action}(\")? %{WORD:}="," fw_action=%{GREEDYDATA:fw_action}"] } } + grok { match => { "msg_all" => [" fwlan=%{DATA:fw_lan} %{WORD:}="," fwlan=%{GREEDYDATA:fw_lan}"] } } + grok { match => { "msg_all" => [" i=%{NUMBER:interval} %{WORD:}="," i=%{GREEDYDATA:interval}"] } } + grok { match => { "msg_all" => [" lic=%{NUMBER:lic} %{WORD:}="," lic=%{GREEDYDATA:lic}"] } } + grok { match => { "msg_all" => [" m=%{NUMBER:event_id} %{WORD:}="," m=%{GREEDYDATA:event_id}"] } } + grok { match => { "msg_all" => [" mailFrom=%{DATA:mailFrom} %{WORD:}="," mailFrom=%{GREEDYDATA:mailFrom}"] } } + grok { match => { "msg_all" => [" n=%{NUMBER:msg_count} %{WORD:}="," n=%{GREEDYDATA:msg_count}"] } } + grok { match => { "msg_all" => [" natDst=%{DATA:nat_dst} %{WORD:}="," natDst=%{GREEDYDATA:nat_dst}"] } } + grok { match => { "msg_all" => [" natDstV6=%{DATA:nat_dst_v6} %{WORD:}="," natDstV6=%{GREEDYDATA:nat_dst_v6}"] } } + grok { match => { "msg_all" => [" natSrc=%{DATA:nat_src} %{WORD:}="," natSrc=%{GREEDYDATA:nat_src}"] } } + grok { match => { "msg_all" => [" natSrcV6=%{DATA:nat_src_v6} %{WORD:}="," natSrcV6=%{GREEDYDATA:nat_src_v6}"] } } + grok { match => { "msg_all" => [" note=(\")?%{DATA:note}(\")? %{WORD:}="," note=%{GREEDYDATA:note}"] } } + grok { match => { "msg_all" => [" npcs=%{DATA:npcs} %{WORD:}="," npcs=%{GREEDYDATA:npcs}"] } } + grok { match => { "msg_all" => [" op=%{NUMBER:request_method} %{WORD:}="," op=%{GREEDYDATA:request_method}"] } } + grok { match => { "msg_all" => [" packetdatId=%{DATA:packetdatId} %{WORD:}="," packetdatId=%{GREEDYDATA:packetdatId}"] } } + grok { match => { "msg_all" => [" packetdatNum=%{DATA:packetdatNum} %{WORD:}="," packetdatNum=%{GREEDYDATA:packetdatNum}"] } } + grok { match => { "msg_all" => [" packetdatEnc=%{DATA:packetdatEnc} %{WORD:}="," packetdatEnc=%{GREEDYDATA:packetdatEnc}"] } } + grok { match => { "msg_all" => [" pri=%{NUMBER:severity} %{WORD:}="," pri=%{GREEDYDATA:severity}"] } } + grok { match => { "msg_all" => [" pt=%{DATA:pt} %{WORD:}="," pt=%{GREEDYDATA:pt}"] } } + grok { match => { "msg_all" => [" rcptTo=%{DATA:email_recipient} %{WORD:}="," rcptTo=%{GREEDYDATA:email_recipient}"] } } + grok { match => { "msg_all" => [" rcvd=%{NUMBER:bytes_rcvd} %{WORD:}="," rcvd=%{GREEDYDATA:bytes_rcvd}"] } } + grok { match => { "msg_all" => [" result=%{NUMBER:result_code} %{WORD:}="," result=%{GREEDYDATA:result_code}"] } } + grok { match => { "msg_all" => [" rpkt=%{NUMBER:pkt_received} %{WORD:}="," rpkt=%{GREEDYDATA:pkt_received}"] } } + grok { match => { "msg_all" => [" rule=(\")?%{DATA:rule}(\")? %{WORD:}="," rule=%{GREEDYDATA:rule}"] } } + grok { match => { "msg_all" => [" sent=%{NUMBER:bytes_sent} %{WORD:}="," sent=%{GREEDYDATA:bytes_sent}"] } } + grok { match => { "msg_all" => [" sess=%{DATA:sess} %{WORD:}="," sess=%{GREEDYDATA:sess}"] } } + grok { match => { "msg_all" => ["(\s)?sn=%{WORD:serial_num} %{WORD:}=","(\s)?sn=%{GREEDYDATA:serial_num}"] } } + grok { match => { "msg_all" => [" spkt=%{NUMBER:pkt_sent} %{WORD:}="," spkt=%{GREEDYDATA:pkt_sent}"] } } + grok { match => { "msg_all" => [" srcMac=%{DATA:srcMac} %{WORD:}="," srcMac=%{GREEDYDATA:srcMac}"] } } + grok { match => { "msg_all" => [" srcZone=%{DATA:srcZone} %{WORD:}="," srcZone=%{GREEDYDATA:srcZone}"] } } + grok { match => { "msg_all" => [" time=(\")?%{DATA:time}(\")? %{WORD:}="," time=%{GREEDYDATA:time}"] } } + grok { match => { "msg_all" => [" type=%{DATA:icmp_type} %{WORD:}="," type=%{GREEDYDATA:icmp_type}"] } } + grok { match => { "msg_all" => [" unsynched=%{DATA:unsynched} %{WORD:}="," unsynched=%{GREEDYDATA:unsynched}"] } } + grok { match => { "msg_all" => [" usestandbysa=%{NUMBER:usestandbysa} %{WORD:}="," usestandbysa=%{GREEDYDATA:usestandbysa}"] } } + grok { match => { "msg_all" => [" (usr|user)=(\")?%{DATA:user}(\")? %{WORD:}="," (usr|user)=%{GREEDYDATA:user}"] } } + grok { match => { "msg_all" => [" vpnpolicy=(\")?%{DATA:vpnpolicy}(\")? %{WORD:}="," vpnpolicy=%{GREEDYDATA:vpnpolicy}"] } } + grok { match => { "msg_all" => [" vpnpolicyDst=(\")?%{DATA:vpnpolicyDst}(\")? %{WORD:}="," vpnpolicyDst=%{GREEDYDATA:vpnpolicyDst}"] } } + + #......................................................................# + #Using grok to parse msg_all components from CEF format only + grok { match => { "msg_all" => ["(\s)?cat=%{NUMBER:msg_category} %{WORD:}="] } } + grok { match => { "msg_all" => [" cn3Label=%{NUMBER:conn_duration} %{WORD:}="," cn3Label=%{GREEDYDATA:conn_duration}"] } } + grok { match => { "msg_all" => [" SWGMSchangeUrl=%{DATA:change} %{WORD:}="," SWGMSchangeUrl=%{GREEDYDATA:change}"] } } + grok { match => { "msg_all" => [" reason=%{NUMBER:code} %{WORD:}="," reason=%{GREEDYDATA:code}"] } } + grok { match => { "msg_all" => [" cn2=%{DATA:icmpCode} %{WORD:}="," cn2=%{GREEDYDATA:icmpCode}"] } } + grok { match => { "msg_all" => [" cs4=%{DATA:interface_statistics} %{WORD:}="," cs4=%{GREEDYDATA:interface_statistics}"] } } + grok { match => { "msg_all" => [" deviceOutboundInterface=%{DATA:device_outbound_interface} %{WORD:}="," deviceOutboundInterface=%{GREEDYDATA:device_outbound_interface}"] } } + grok { match => { "msg_all" => [" deviceInboundInterface=%{DATA:device_inbound_interface} %{WORD:}="," deviceInboundInterface=%{GREEDYDATA:device_inbound_interface}"] } } + grok { match => { "msg_all" => [" dpt=%{NUMBER:dest_port} %{WORD:}="," dpt=%{GREEDYDATA:dest_port}"] } } + grok { match => { "msg_all" => [" dnpt=%{NUMBER:nat_dest_port} %{WORD:}="," dnpt=%{GREEDYDATA:nat_dest_port}"] } } + grok { match => { "msg_all" => [" dmac=%{DATA:dstMac} %{WORD:}="," dmac=%{GREEDYDATA:dstMac}"] } } + grok { match => { "msg_all" => [" request=%{DATA:request} %{WORD:}="," request=%{GREEDYDATA:request}"] } } + grok { match => { "msg_all" => [" cs4Label=%{DATA:dstZone} %{WORD:}="," cs4Label=%{GREEDYDATA:dstZone}"] } } + grok { match => { "msg_all" => [" cs6label=%{NUMBER:session_dur} %{WORD:}="," cs6label=%{GREEDYDATA:session_dur}"] } } + grok { match => { "msg_all" => [" flowType=%{NUMBER:flow_type} %{WORD:}="," flowType=%{GREEDYDATA:flow_type}"] } } + grok { match => { "msg_all" => [" cnt=%{NUMBER:msg_count} %{WORD:}="," cnt=%{GREEDYDATA:msg_count}"] } } + grok { match => { "msg_all" => [" cs2Label=%{DATA:nat_dst} %{WORD:}="," cs2Label=%{GREEDYDATA:nat_dst}"] } } + grok { match => { "msg_all" => [" cs1Label=%{DATA:nat_src} %{WORD:}="," cs1Label=%{GREEDYDATA:nat_src}"] } } + grok { match => { "msg_all" => [" cs6=(\")?%{DATA:note}(\")? %{WORD:}="," cs6=%{GREEDYDATA:note}"] } } + grok { match => { "msg_all" => [" cs5=%{DATA:npcs} %{WORD:}="," cs5=%{GREEDYDATA:npcs}"] } } + grok { match => { "msg_all" => [" requestMethod=%{NUMBER:request_method} %{WORD:}="," requestMethod=%{GREEDYDATA:request_method}"] } } + grok { match => { "msg_all" => [" in=%{NUMBER:bytes_rcvd} %{WORD:}="," in=%{GREEDYDATA:bytes_rcvd}"] } } + grok { match => { "msg_all" => [" outcome=%{NUMBER:result_code} %{WORD:}="," outcome=%{GREEDYDATA:result_code}"] } } + grok { match => { "msg_all" => [" cn1Label=%{NUMBER:pkt_received} %{WORD:}="," cn1Label=%{GREEDYDATA:pkt_received}"] } } + grok { match => { "msg_all" => [" cs1=(\")?%{DATA:rule}(\")? %{WORD:}="," cs1=%{GREEDYDATA:rule}"] } } + grok { match => { "msg_all" => [" out=%{NUMBER:bytes_sent} %{WORD:}="," out=%{GREEDYDATA:bytes_sent}"] } } + grok { match => { "msg_all" => [" cs5Label=%{DATA:sess} %{WORD:}="," cs5Label=%{GREEDYDATA:sess}"] } } + grok { match => { "msg_all" => [" cn2Label=%{NUMBER:pkt_sent} %{WORD:}="," cn2Label=%{GREEDYDATA:pkt_sent}"] } } + grok { match => { "msg_all" => [" spt=%{NUMBER:src_port} %{WORD:}="," spt=%{GREEDYDATA:src_port}"] } } + grok { match => { "msg_all" => [" snpt=%{NUMBER:nat_src_port} %{WORD:}="," snpt=%{GREEDYDATA:nat_src_port}"] } } + grok { match => { "msg_all" => [" smac=%{DATA:srcMac} %{WORD:}="," smac=%{GREEDYDATA:srcMac}"] } } + grok { match => { "msg_all" => [" cs3Label=%{DATA:srcZone} %{WORD:}="," cs3Label=%{GREEDYDATA:srcZone}"] } } + grok { match => { "msg_all" => [" cn1=%{DATA:icmp_type} %{WORD:}="," cn1=%{GREEDYDATA:icmp_type}"] } } + grok { match => { "msg_all" => [" susr=(\")?%{DATA:user}(\")? %{WORD:}="," susr=%{GREEDYDATA:user}"] } } + grok { match => { "msg_all" => [" cs2=%{DATA:vpnpolicy} %{WORD:}="," cs2=%{GREEDYDATA:vpnpolicy}"] } } + grok { match => { "msg_all" => [" cs3=%{DATA:vpnpolicyDst} %{WORD:}="," cs3=%{GREEDYDATA:vpnpolicyDst}"] } } + + } + + # Clean double quotes from extracted fields + mutate { + gsub => [ + "fw_action", "^\"", "", + "fw_action", "\"$", "", + "msg", "^\"", "", + "msg", "\"$", "", + "note", "^\"", "", + "note", "\"$", "", + "rule", "^\"", "", + "rule", "\"$", "", + "user", "^\"", "", + "user", "\"$", "", + "vpnpolicy", "^\"", "", + "vpnpolicy", "\"$", "", + "vpnpolicyDst", "^\"", "", + "vpnpolicyDst", "\"$", "", + "fileid", "^\"", "", + "fileid", "\"$", "", + "referer", "^\"", "", + "referer", "\"$", "", + "time", "^\"", "", + "time", "\"$", "", + "cfs_category", "^\"", "", + "cfs_category", "\"$", "", + "ipscat", "^\"", "", + "ipscat", "\"$", "", + "app_cat", "^\"", "", + "app_cat", "\"$", "", + "appid", "^\"", "", + "appid", "\"$", "", + "arg", "^\"", "", + "arg", "\"$", "", + "app_name", "^\"", "", + "app_name", "\"$", "", + "contentObject", "^\"", "", + "contentObject", "\"$", "", + "request", "^\"", "", + "request", "\"$", "", + "anti_spyware_cat", "^\"", "", + "anti_spyware_cat", "\"$", "", + "anti_spyware_pri", "^\"", "", + "anti_spyware_pri", "\"$", "", + "mgmt_source_ip", "^\"", "", + "mgmt_source_ip", "\"$", "", + "radio", "^\"", "", + "radio", "\"$", "", + "station", "^\"", "", + "station", "\"$", "" + ] + } + #Checking if the src field exists and using grok to parse it + if ([src]){ + grok { + match => { "src" => "%{IPORHOST:src_ip}(:%{NUMBER:src_port})?(:%{WORD:device_inbound_interface})?(:%{GREEDYDATA:src_resolved_name})?"} + } + } + #Checking if the dst field exists and using grok to parse it + if ([dst]){ + grok { + match => { "dst" => "%{IPORHOST:dest_ip}(:%{NUMBER:dest_port})?(:%{WORD:device_outbound_interface})?(:%{GREEDYDATA:dest_resolved_name})?"} + } + } + #Checking if the protocol field exists and using grok to parse it + if ([protocol]){ + grok { + match => { "protocol" => "%{WORD:proto}(/%{GREEDYDATA:service})?"} + } + } + #......................................................................# + #Generate UTM ACTION field when connection established + #......................................................................# + if ([event_id] and ( + ( ([event_id]=="16") or ([event_id]=="24") or ([event_id]=="29") or ([event_id]=="31") or ([event_id]=="36") or ([event_id]=="43") or ([event_id]=="53") or ([event_id]=="58") or ([event_id]=="87") or ([event_id]=="88") + or ([event_id]=="89") or ([event_id]=="98") or ([event_id]=="131") or ([event_id]=="132") or ([event_id]=="168") or ([event_id]=="199") or ([event_id]=="205") or ([event_id]=="207") or ([event_id]=="215") + or ([event_id]=="235") or ([event_id]=="236") or ([event_id]=="237") or ([event_id]=="238") or ([event_id]=="261") or ([event_id]=="262") or ([event_id]=="263") or ([event_id]=="264") or ([event_id]=="265") + or ([event_id]=="273") or ([event_id]=="286") or ([event_id]=="288") or ([event_id]=="289") or ([event_id]=="307") or ([event_id]=="318") or ([event_id]=="319") or ([event_id]=="327") or ([event_id]=="335") + or ([event_id]=="346") or ([event_id]=="351") or ([event_id]=="352") or ([event_id]=="353") or ([event_id]=="354") or ([event_id]=="355") or ([event_id]=="356") or ([event_id]=="357") or ([event_id]=="358") + or ([event_id]=="372") or ([event_id]=="373") or ([event_id]=="375") or ([event_id]=="378") or ([event_id]=="379") or ([event_id]=="381") or ([event_id]=="388") or ([event_id]=="389") or ([event_id]=="390") + or ([event_id]=="391") or ([event_id]=="392") or ([event_id]=="393") or ([event_id]=="394") or ([event_id]=="395") or ([event_id]=="396") or ([event_id]=="429") or ([event_id]=="430") or ([event_id]=="431") + or ([event_id]=="432") or ([event_id]=="433") or ([event_id]=="445") or ([event_id]=="429") or ([event_id]=="535") or ([event_id]=="537") or ([event_id]=="542") or ([event_id]=="622") or ([event_id]=="623") + or ([event_id]=="634") or ([event_id]=="636") or ([event_id]=="646") or ([event_id]=="647") or ([event_id]=="651") or ([event_id]=="666") or ([event_id]=="712") or ([event_id]=="713") or ([event_id]=="734") + or ([event_id]=="735") or ([event_id]=="760") or ([event_id]=="797") or ([event_id]=="798") or ([event_id]=="820") or ([event_id]=="859") or ([event_id]=="891") or ([event_id]=="892") or ([event_id]=="1008") + or ([event_id]=="1027") or ([event_id]=="1028") or ([event_id]=="1080") or ([event_id]=="1087") or ([event_id]=="1091") or ([event_id]=="1092") or ([event_id]=="1112") or ([event_id]=="1114") or ([event_id]=="1116") + or ([event_id]=="1124") or ([event_id]=="1153") or ([event_id]=="1179") or ([event_id]=="1183") or ([event_id]=="1187") or ([event_id]=="1267") or ([event_id]=="1424") or ([event_id]=="1445") or ([event_id]=="1452") + or ([event_id]=="1490") or ([event_id]=="1491") or ([event_id]=="1498") or ([event_id]=="1500") or ([event_id]=="1502") or ([event_id]=="1503") or ([event_id]=="1504") or ([event_id]=="1534")) + or ((([event_id]!="16") and ([event_id]!="24") and ([event_id]!="29") and ([event_id]!="31") and ([event_id]!="36") and ([event_id]!="43") and ([event_id]!="53") and ([event_id]!="58") and ([event_id]!="87") and ([event_id]!="88") + and ([event_id]!="89") and ([event_id]!="98") and ([event_id]!="131") and ([event_id]!="132") and ([event_id]!="168") and ([event_id]!="199") and ([event_id]!="205") and ([event_id]!="207") and ([event_id]!="215") + and ([event_id]!="235") and ([event_id]!="236") and ([event_id]!="237") and ([event_id]!="238") and ([event_id]!="261") and ([event_id]!="262") and ([event_id]!="263") and ([event_id]!="264") and ([event_id]!="265") + and ([event_id]!="273") and ([event_id]!="286") and ([event_id]!="288") and ([event_id]!="289") and ([event_id]!="307") and ([event_id]!="318") and ([event_id]!="319") and ([event_id]!="327") and ([event_id]!="335") + and ([event_id]!="346") and ([event_id]!="351") and ([event_id]!="352") and ([event_id]!="353") and ([event_id]!="354") and ([event_id]!="355") and ([event_id]!="356") and ([event_id]!="357") and ([event_id]!="358") + and ([event_id]!="372") and ([event_id]!="373") and ([event_id]!="375") and ([event_id]!="378") and ([event_id]!="379") and ([event_id]!="381") and ([event_id]!="388") and ([event_id]!="389") and ([event_id]!="390") + and ([event_id]!="391") and ([event_id]!="392") and ([event_id]!="393") and ([event_id]!="394") and ([event_id]!="395") and ([event_id]!="396") and ([event_id]!="429") and ([event_id]!="430") and ([event_id]!="431") + and ([event_id]!="432") and ([event_id]!="433") and ([event_id]!="445") and ([event_id]!="429") and ([event_id]!="535") and ([event_id]!="537") and ([event_id]!="542") and ([event_id]!="622") and ([event_id]!="623") + and ([event_id]!="634") and ([event_id]!="636") and ([event_id]!="646") and ([event_id]!="647") and ([event_id]!="651") and ([event_id]!="666") and ([event_id]!="712") and ([event_id]!="713") and ([event_id]!="734") + and ([event_id]!="735") and ([event_id]!="760") and ([event_id]!="797") and ([event_id]!="798") and ([event_id]!="820") and ([event_id]!="859") and ([event_id]!="891") and ([event_id]!="892") and ([event_id]!="1008") + and ([event_id]!="1027") and ([event_id]!="1028") and ([event_id]!="1080") and ([event_id]!="1087") and ([event_id]!="1091") and ([event_id]!="1092") and ([event_id]!="1112") and ([event_id]!="1114") and ([event_id]!="1116") + and ([event_id]!="1124") and ([event_id]!="1153") and ([event_id]!="1179") and ([event_id]!="1183") and ([event_id]!="1187") and ([event_id]!="1267") and ([event_id]!="1424") and ([event_id]!="1445") and ([event_id]!="1452") + and ([event_id]!="1490") and ([event_id]!="1491") and ([event_id]!="1498") and ([event_id]!="1500") and ([event_id]!="1502") and ([event_id]!="1503") and ([event_id]!="1504") and ([event_id]!="1534")) + and ([src_ip]) and ([dest_ip]) and ([bytes_rcvd]) and ([bytes_rcvd]!="0") and (([bytes_sent] and ([bytes_sent]!="0")) or ([proto] and ([proto]=="ftp")))))){ + mutate { + add_field => { "[logx][utm][action]" => "Success" } + } + } + + mutate { + #Rename fields used by correlation engine + rename => { "[src_ip]" => "[logx][sonicwall][src_ip]" } + rename => { "[src_port]" => "[logx][sonicwall][src_port]" } + rename => { "[dest_ip]" => "[logx][sonicwall][dest_ip]" } + rename => { "[dest_port]" => "[logx][sonicwall][dest_port]" } + rename => { "[proto]" => "[logx][sonicwall][proto]" } + + #Rename all others fields + rename => { "[device_inbound_interface]" => "[logx][sonicwall][device_inbound_interface]" } + rename => { "[device_outbound_interface]" => "[logx][sonicwall][device_outbound_interface]" } + rename => { "[src_resolved_name]" => "[logx][sonicwall][src_resolved_name]" } + rename => { "[dest_resolved_name]" => "[logx][sonicwall][dest_resolved_name]" } + rename => { "[af_policy_id]" => "[logx][sonicwall][af_policy_id]" } + rename => { "[af_policy]" => "[logx][sonicwall][af_policy]" } + rename => { "[af_policy_type]" => "[logx][sonicwall][af_policy_type]" } + rename => { "[af_policy_service]" => "[logx][sonicwall][af_policy_service]" } + rename => { "[af_policy_action]" => "[logx][sonicwall][af_policy_action]" } + rename => { "[af_policy_object]" => "[logx][sonicwall][af_policy_object]" } + rename => { "[active_interface]" => "[logx][sonicwall][active_interface]" } + rename => { "[num_app_id]" => "[logx][sonicwall][num_app_id]" } + rename => { "[app_cat]" => "[logx][sonicwall][app_cat]" } + rename => { "[appid]" => "[logx][sonicwall][appid]" } + rename => { "[app_name]" => "[logx][sonicwall][app_name]" } + rename => { "[arg]" => "[logx][sonicwall][arg]" } + rename => { "[bcast_pkt_received]" => "[logx][sonicwall][bcast_pkt_received]" } + rename => { "[bcast_pkt_transm]" => "[logx][sonicwall][bcast_pkt_transm]" } + rename => { "[blade_id]" => "[logx][sonicwall][blade_id]" } + rename => { "[msg_category]" => "[logx][sonicwall][msg_category]" } + rename => { "[cfs_category]" => "[logx][sonicwall][cfs_category]" } + rename => { "[rule_category]" => "[logx][sonicwall][rule_category]" } + rename => { "[conn_duration]" => "[logx][sonicwall][conn_duration]" } + rename => { "[change]" => "[logx][sonicwall][change]" } + rename => { "[code]" => "[logx][sonicwall][code]" } + rename => { "[conns]" => "[logx][sonicwall][conns]" } + rename => { "[contentObject]" => "[logx][sonicwall][contentObject]" } + rename => { "[deep_packet_inspection]" => "[logx][sonicwall][deep_packet_inspection]" } + rename => { "[dstMac]" => "[logx][sonicwall][destMac]" } + rename => { "[dstV6]" => "[logx][sonicwall][destV6]" } + rename => { "[dstname]" => "[logx][sonicwall][destname]" } + rename => { "[dstZone]" => "[logx][sonicwall][destZone]" } + rename => { "[session_dur]" => "[logx][sonicwall][session_dur]" } + rename => { "[fw_status_report]" => "[logx][sonicwall][fw_status_report]" } + rename => { "[flow_type]" => "[logx][sonicwall][flow_type]" } + rename => { "[fileid]" => "[logx][sonicwall][fileid]" } + rename => { "[filetxstatus]" => "[logx][sonicwall][filetxstatus]" } + rename => { "[fw_wan]" => "[logx][sonicwall][fw_wan]" } + rename => { "[fw_action]" => "[logx][sonicwall][fw_action]" } + rename => { "[fw_lan]" => "[logx][sonicwall][fw_lan]" } + rename => { "[group_cat]" => "[logx][sonicwall][group_cat]" } + rename => { "[good_bytes_received]" => "[logx][sonicwall][good_bytes_received]" } + rename => { "[good_bytes_transm]" => "[logx][sonicwall][good_bytes_transm]" } + rename => { "[id]" => "[logx][sonicwall][id]"} + rename => { "[interval]" => "[logx][sonicwall][interval]" } + rename => { "[icmpCode]" => "[logx][sonicwall][icmpCode]" } + rename => { "[interface_statistics_reported]" => "[logx][sonicwall][interface_statistics_reported]" } + rename => { "[ipscat]" => "[logx][sonicwall][ipscat]" } + rename => { "[ipspri]" => "[logx][sonicwall][ipspri]" } + rename => { "[lic]" => "[logx][sonicwall][lic]" } + rename => { "[event_id]" => "[logx][sonicwall][event_id]" } + rename => { "[mailFrom]" => "[logx][sonicwall][mailFrom]" } + rename => { "[msg]" => "[logx][sonicwall][msg]" } + rename => { "[mgmt_source_ip]" => "[logx][sonicwall][mgmt_source_ip]" } + rename => { "[msg_count]" => "[logx][sonicwall][msg_count]" } + rename => { "[nat_dst]" => "[logx][sonicwall][nat_dest]" } + rename => { "[nat_dst_v6]" => "[logx][sonicwall][nat_dest_v6]" } + rename => { "[nat_src]" => "[logx][sonicwall][nat_src]" } + rename => { "[nat_src_v6]" => "[logx][sonicwall][nat_src_v6]" } + rename => { "[note]" => "[logx][sonicwall][note]" } + rename => { "[npcs]" => "[logx][sonicwall][npcs]" } + rename => { "[request_method]" => "[logx][sonicwall][request_method]" } + rename => { "[packetdatId]" => "[logx][sonicwall][packetdatId]" } + rename => { "[packetdatNum]" => "[logx][sonicwall][packetdatNum]" } + rename => { "[packetdatEnc]" => "[logx][sonicwall][packetdatEnc]" } + rename => { "[severity]" => "[logx][sonicwall][severity]" } + rename => { "[pt]" => "[logx][sonicwall][pt]" } + rename => { "[email_recipient]" => "[logx][sonicwall][email_recipient]" } + rename => { "[radio]" => "[logx][sonicwall][radio]" } + rename => { "[bytes_rcvd]" => "[logx][sonicwall][bytes_rcvd]" } + rename => { "[referer]" => "[logx][sonicwall][referer]" } + rename => { "[result_code]" => "[logx][sonicwall][result_code]" } + rename => { "[pkt_received]" => "[logx][sonicwall][pkt_received]" } + rename => { "[rule]" => "[logx][sonicwall][rule]" } + rename => { "[bytes_sent]" => "[logx][sonicwall][bytes_sent]" } + rename => { "[service]" => "[logx][sonicwall][service]" } + rename => { "[sess]" => "[logx][sonicwall][sess]" } + rename => { "[sid]" => "[logx][sonicwall][sid]" } + rename => { "[serial_num]" => "[logx][sonicwall][serial_num]" } + rename => { "[pkt_sent]" => "[logx][sonicwall][pkt_sent]" } + rename => { "[priority]" => "[logx][sonicwall][priority]" } + rename => { "[anti_spyware_cat]" => "[logx][sonicwall][anti_spyware_cat]" } + rename => { "[anti_spyware_pri]" => "[logx][sonicwall][anti_spyware_pri]" } + rename => { "[srcMac]" => "[logx][sonicwall][srcMac]" } + rename => { "[srcZone]" => "[logx][sonicwall][srcZone]" } + rename => { "[station]" => "[logx][sonicwall][station]" } + rename => { "[time]" => "[logx][sonicwall][time]" } + rename => { "[icmp_type]" => "[logx][sonicwall][icmp_type]" } + rename => { "[ucast_pkt_received]" => "[logx][sonicwall][ucast_pkt_received]" } + rename => { "[ucast_pkt_transm]" => "[logx][sonicwall][ucast_pkt_transm]" } + rename => { "[unsynched]" => "[logx][sonicwall][unsynched]" } + rename => { "[usestandbysa]" => "[logx][sonicwall][usestandbysa]" } + rename => { "[user]" => "[logx][sonicwall][user]" } + rename => { "[vpnpolicy]" => "[logx][sonicwall][vpnpolicy]" } + rename => { "[vpnpolicyDst]" => "[logx][sonicwall][vpnpolicyDst]" } + + #Rename fields of CEF format only + rename => { "[nat_dest_port]" => "[logx][sonicwall][nat_dest_port]" } + rename => { "[request]" => "[logx][sonicwall][request]" } + rename => { "[nat_src_port]" => "[logx][sonicwall][nat_src_port]" } + rename => { "[cef_version]" => "[logx][sonicwall][cef_version]" } + rename => { "[dvc_vendor]" => "[logx][sonicwall][dvc_vendor]" } + rename => { "[dvc_product]" => "[logx][sonicwall][dvc_product]" } + rename => { "[dvc_version]" => "[logx][sonicwall][dvc_version]" } + rename => { "[event_name]" => "[logx][sonicwall][event_name]" } + } + + #Finally, remove unnecessary fields + mutate { + remove_field => ["@version","path","tags","type","syslog_version", + "not_defined","src","protocol","dst","msg_all","cef_msg_all","syslog_date_host"] + } + } + } + #Also, remove unwanted fields if the message not match with conditions + mutate { + remove_field => ["original_log_message","headers"] + } +}' + WHERE id=1511; + ]]> + + + \ No newline at end of file diff --git a/backend/src/main/resources/config/liquibase/master.xml b/backend/src/main/resources/config/liquibase/master.xml index 72ec78fe6..f6ce831a6 100644 --- a/backend/src/main/resources/config/liquibase/master.xml +++ b/backend/src/main/resources/config/liquibase/master.xml @@ -109,5 +109,9 @@ + + + + From b446b218626ebc4db90bc45b5be36b596cff6d18 Mon Sep 17 00:00:00 2001 From: Yorjander Hernandez Vergara <99102374+Kbayero@users.noreply.github.com> Date: Mon, 10 Nov 2025 10:26:59 -0500 Subject: [PATCH 18/18] Update CHANGELOG.md --- CHANGELOG.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e33dff40..76454990c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # UTMStack 10.9.2 Release Notes --- Added new Pacific time zones (New Zealand and Fiji) to the Date Settings section. --- Added TLS connection options and setup steps for secure Syslog integration --- Improved sorting of asset sources in tables, ensuring consistent and predictable order for names, IPs, and combined entries. -–- Improved correlation rule handling for pfSense and SonicWall data sources to enhance detection accuracy and event normalization. \ No newline at end of file +- Added new Pacific time zones (New Zealand and Fiji) to the Date Settings section. +- Added TLS connection options and setup steps for secure Syslog integration +- Improved sorting of asset sources in tables, ensuring consistent and predictable order for names, IPs, and combined entries. +– Improved correlation rule handling for pfSense and SonicWall data sources to enhance detection accuracy and event normalization.