From 0d34bf7d3cafd627a580c20d7bc74576d8c18688 Mon Sep 17 00:00:00 2001 From: marco370 <48531002-marco370@users.noreply.replit.com> Date: Mon, 17 Nov 2025 17:35:37 +0000 Subject: [PATCH] Update log parsing to better identify network traffic and DDoS events Refactors the `SyslogParser` class in `python_ml/syslog_parser.py` to use a new, more comprehensive regex pattern (`main_pattern`) for parsing MikroTik logs. This includes improved identification of 'forward' and 'detected-ddos forward' actions, protocol details (UDP, TCP, ICMP), and associated IP addresses, ports, and lengths. The changes aim to accurately capture network traffic and potential DDoS events from MikroTik logs. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: b7377ada-e722-475a-86d2-07f21299ec70 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/MkBJZ0L --- ...out-sfp-sf-1763400878278_1763400878278.txt | 42 ++++++++++ python_ml/syslog_parser.py | 79 ++++++++++--------- 2 files changed, 82 insertions(+), 39 deletions(-) create mode 100644 attached_assets/Pasted-head-20-var-log-mikrotik-raw-log-Nov-17-16-52-16-FIBRA-forward-in-sfp-sfpplus1-VS-FTTO-out-sfp-sf-1763400878278_1763400878278.txt diff --git a/attached_assets/Pasted-head-20-var-log-mikrotik-raw-log-Nov-17-16-52-16-FIBRA-forward-in-sfp-sfpplus1-VS-FTTO-out-sfp-sf-1763400878278_1763400878278.txt b/attached_assets/Pasted-head-20-var-log-mikrotik-raw-log-Nov-17-16-52-16-FIBRA-forward-in-sfp-sfpplus1-VS-FTTO-out-sfp-sf-1763400878278_1763400878278.txt new file mode 100644 index 0000000..fa2ceee --- /dev/null +++ b/attached_assets/Pasted-head-20-var-log-mikrotik-raw-log-Nov-17-16-52-16-FIBRA-forward-in-sfp-sfpplus1-VS-FTTO-out-sfp-sf-1763400878278_1763400878278.txt @@ -0,0 +1,42 @@ +head -20 /var/log/mikrotik/raw.log +Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 +Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 +Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 +Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:9991->185.203.26.77:53, len 65 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:9991->185.203.26.77:53, len 65 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:43863->185.203.26.34:8472, len 210 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:43863->185.203.26.34:8472, len 210 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56224->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56224->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56225->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56225->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:58268->172.67.143.237:443, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:58268->172.67.143.237:443, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56676->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.233:35832->192.168.25.254:80, len 60 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:56670->185.203.26.34:8472, len 178 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:56670->185.203.26.34:8472, len 178 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 72.46.85.161:43970->185.203.24.135:51688, len 44 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 72.46.85.161:43970->185.203.24.135:51688, len 44 +[root@ids python_ml]# tail -20 /var/log/mikrotik/raw.log +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new src-mac 98:da:c4:75:8c:fb, proto UDP, 10.0.254.170:56065->104.20.23.252:443, len 1278 +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new src-mac 98:da:c4:75:8c:fb, proto UDP, 10.0.254.170:56065->104.20.23.252:443, len 1278 +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new,snat src-mac 98:da:c4:75:8c:fb, proto UDP, 10.0.254.170:56065->104.20.23.252:443, NAT (10.0.254.170:56065->185.203.27.253:56065)->104.20.23.252:443, len 1278 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 126.220.199.81:32730->185.203.25.204:53, len 82 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 126.220.199.81:32730->185.203.25.204:53, len 82 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 160.202.129.17:43994->185.203.24.15:56929, len 44 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 160.202.129.17:43994->185.203.24.15:56929, len 44 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 95.216.123.229:4653->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 95.216.123.229:4653->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:28065->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:28065->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 168.227.31.21:59518->185.203.25.204:53, len 63 +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 10.0.254.242:47946->3.223.194.130:443, len 60 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 168.227.31.21:59518->185.203.25.204:53, len 63 +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 10.0.254.242:47946->3.223.194.130:443, len 60 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:3117->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:3117->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:30733->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:30733->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 35.203.211.209:50481->185.203.24.138:27482, len 44 \ No newline at end of file diff --git a/python_ml/syslog_parser.py b/python_ml/syslog_parser.py index 8298562..c1a7f51 100644 --- a/python_ml/syslog_parser.py +++ b/python_ml/syslog_parser.py @@ -23,25 +23,17 @@ class SyslogParser: self.conn = None self.cursor = None - # Pattern regex per parsare log MikroTik - # Formato: timestamp hostname tag: message - self.patterns = { - # Firewall connection - 'firewall': re.compile( - r'(?Paccept|drop|reject).*' - r'src-address=(?P[\d.]+):(?P\d+).*' - r'dst-address=(?P[\d.]+):(?P\d+).*' - r'proto=(?P\w+).*' - r'(?:len=(?P\d+))?' - ), - # Connection tracking - 'connection': re.compile( - r'(?P[\d.]+):(?P\d+)->(?P[\d.]+):(?P\d+).*' - r'proto (?P\w+).*' - r'(?:packets: (?P\d+))?.*' - r'(?:bytes: (?P\d+))?' - ), - } + # Pattern regex per parsare log MikroTik (formato reale) + # Esempio: Nov 17 16:52:16 FIBRA forward: ... proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 + # Esempio: Nov 17 16:52:16 FIBRA detected-ddos forward: ... proto TCP (SYN), 82.62.84.108:43863->185.203.26.34:8472, len 210 + + self.main_pattern = re.compile( + r'(?Pforward|detected-ddos forward):.*?' + r'proto (?PUDP|TCP|ICMP)(?:\s+\((?P[^)]+)\))?.*?' + r'(?P[\d.]+):(?P\d+)->(?P[\d.]+):(?P\d+).*?' + r'len (?P\d+)', + re.IGNORECASE + ) def connect_db(self): """Connessione al database PostgreSQL""" @@ -65,9 +57,13 @@ class SyslogParser: """ Analizza una singola riga di log MikroTik Returns: Dict con dati parsati o None se non parsabile + + Formato reale: + Nov 17 16:52:16 FIBRA forward: in:... proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 + Nov 17 16:52:16 FIBRA detected-ddos forward: ... proto TCP (SYN), 82.62.84.108:43863->185.203.26.34:8472, len 210 """ - # Estrai timestamp, hostname, tag e messaggio - # Formato: Jan 15 10:30:45 router1 firewall,info: drop src-address=... + # Estrai timestamp, hostname, messaggio + # Formato: Nov 17 16:52:16 FIBRA forward: ... parts = line.split(None, 4) if len(parts) < 5: return None @@ -84,24 +80,29 @@ class SyslogParser: except ValueError: return None - # Prova pattern firewall - for pattern_name, pattern in self.patterns.items(): - match = pattern.search(message) - if match: - data = match.groupdict() - - # Aggiungi metadati - data['timestamp'] = timestamp - data['router_name'] = hostname - data['log_type'] = pattern_name - data['raw_message'] = message.strip() - - # Converti numeri - for key in ['src_port', 'dst_port', 'len', 'packets', 'bytes']: - if key in data and data[key]: - data[key] = int(data[key]) - - return data + # Match pattern principale + match = self.main_pattern.search(message) + if match: + data = match.groupdict() + + # Aggiungi metadati + data['timestamp'] = timestamp + data['router_name'] = hostname + data['raw_message'] = line.strip() + + # Determina action finale + action = data['action'] + if 'detected-ddos' in action: + data['action'] = 'ddos' + else: + data['action'] = 'forward' + + # Converti numeri + for key in ['src_port', 'dst_port', 'len']: + if key in data and data[key]: + data[key] = int(data[key]) + + return data return None