Update log parsing to better identify network traffic and DDoS events
Refactors the `SyslogParser` class in `python_ml/syslog_parser.py` to use a new, more comprehensive regex pattern (`main_pattern`) for parsing MikroTik logs. This includes improved identification of 'forward' and 'detected-ddos forward' actions, protocol details (UDP, TCP, ICMP), and associated IP addresses, ports, and lengths. The changes aim to accurately capture network traffic and potential DDoS events from MikroTik logs. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: b7377ada-e722-475a-86d2-07f21299ec70 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/MkBJZ0L
This commit is contained in:
marco370
parent
51aa026aae
commit
0d34bf7d3c
@ -0,0 +1,42 @@
|
|||||||
|
head -20 /var/log/mikrotik/raw.log
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280
|
||||||
|
Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:9991->185.203.26.77:53, len 65
|
||||||
|
Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:9991->185.203.26.77:53, len 65
|
||||||
|
Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:43863->185.203.26.34:8472, len 210
|
||||||
|
Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:43863->185.203.26.34:8472, len 210
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:<pppoe-1018_mario.alfieri> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56224->172.67.143.237:80, len 60
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:<pppoe-1018_mario.alfieri> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56224->172.67.143.237:80, len 60
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:<pppoe-1018_mario.alfieri> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56225->172.67.143.237:80, len 60
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:<pppoe-1018_mario.alfieri> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56225->172.67.143.237:80, len 60
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:<pppoe-1018_mario.alfieri> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:58268->172.67.143.237:443, len 60
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:<pppoe-1018_mario.alfieri> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:58268->172.67.143.237:443, len 60
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:<pppoe-1018_mario.alfieri> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56676->172.67.143.237:80, len 60
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:<pppoe-caronte.hightek_01> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.233:35832->192.168.25.254:80, len 60
|
||||||
|
Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:56670->185.203.26.34:8472, len 178
|
||||||
|
Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:56670->185.203.26.34:8472, len 178
|
||||||
|
Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 72.46.85.161:43970->185.203.24.135:51688, len 44
|
||||||
|
Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 72.46.85.161:43970->185.203.24.135:51688, len 44
|
||||||
|
[root@ids python_ml]# tail -20 /var/log/mikrotik/raw.log
|
||||||
|
Nov 17 18:34:26 FIBRA forward: in:<pppoe-023_maria.barba> out:sfp-sfpplus2_VS_AS, connection-state:new src-mac 98:da:c4:75:8c:fb, proto UDP, 10.0.254.170:56065->104.20.23.252:443, len 1278
|
||||||
|
Nov 17 18:34:26 FIBRA forward: in:<pppoe-023_maria.barba> out:sfp-sfpplus2_VS_AS, connection-state:new src-mac 98:da:c4:75:8c:fb, proto UDP, 10.0.254.170:56065->104.20.23.252:443, len 1278
|
||||||
|
Nov 17 18:34:26 FIBRA forward: in:<pppoe-023_maria.barba> out:sfp-sfpplus2_VS_AS, connection-state:new,snat src-mac 98:da:c4:75:8c:fb, proto UDP, 10.0.254.170:56065->104.20.23.252:443, NAT (10.0.254.170:56065->185.203.27.253:56065)->104.20.23.252:443, len 1278
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-gaetano.dibenedetto>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 126.220.199.81:32730->185.203.25.204:53, len 82
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-gaetano.dibenedetto>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 126.220.199.81:32730->185.203.25.204:53, len 82
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 160.202.129.17:43994->185.203.24.15:56929, len 44
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 160.202.129.17:43994->185.203.24.15:56929, len 44
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 95.216.123.229:4653->185.203.26.77:53, len 65
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 95.216.123.229:4653->185.203.26.77:53, len 65
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:28065->185.203.26.77:53, len 65
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:28065->185.203.26.77:53, len 65
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-gaetano.dibenedetto>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 168.227.31.21:59518->185.203.25.204:53, len 63
|
||||||
|
Nov 17 18:34:26 FIBRA forward: in:<pppoe-1099_maddalena.esposito> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 10.0.254.242:47946->3.223.194.130:443, len 60
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-gaetano.dibenedetto>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 168.227.31.21:59518->185.203.25.204:53, len 63
|
||||||
|
Nov 17 18:34:26 FIBRA forward: in:<pppoe-1099_maddalena.esposito> out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 10.0.254.242:47946->3.223.194.130:443, len 60
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:3117->185.203.26.77:53, len 65
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:3117->185.203.26.77:53, len 65
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:30733->185.203.26.77:53, len 65
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:<pppoe-571_alberto.apostolico>, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:30733->185.203.26.77:53, len 65
|
||||||
|
Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 35.203.211.209:50481->185.203.24.138:27482, len 44
|
||||||
@ -23,25 +23,17 @@ class SyslogParser:
|
|||||||
self.conn = None
|
self.conn = None
|
||||||
self.cursor = None
|
self.cursor = None
|
||||||
|
|
||||||
# Pattern regex per parsare log MikroTik
|
# Pattern regex per parsare log MikroTik (formato reale)
|
||||||
# Formato: timestamp hostname tag: message
|
# Esempio: Nov 17 16:52:16 FIBRA forward: ... proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280
|
||||||
self.patterns = {
|
# Esempio: Nov 17 16:52:16 FIBRA detected-ddos forward: ... proto TCP (SYN), 82.62.84.108:43863->185.203.26.34:8472, len 210
|
||||||
# Firewall connection
|
|
||||||
'firewall': re.compile(
|
self.main_pattern = re.compile(
|
||||||
r'(?P<action>accept|drop|reject).*'
|
r'(?P<action>forward|detected-ddos forward):.*?'
|
||||||
r'src-address=(?P<src_ip>[\d.]+):(?P<src_port>\d+).*'
|
r'proto (?P<proto>UDP|TCP|ICMP)(?:\s+\((?P<tcp_flags>[^)]+)\))?.*?'
|
||||||
r'dst-address=(?P<dst_ip>[\d.]+):(?P<dst_port>\d+).*'
|
r'(?P<src_ip>[\d.]+):(?P<src_port>\d+)->(?P<dst_ip>[\d.]+):(?P<dst_port>\d+).*?'
|
||||||
r'proto=(?P<proto>\w+).*'
|
r'len (?P<len>\d+)',
|
||||||
r'(?:len=(?P<len>\d+))?'
|
re.IGNORECASE
|
||||||
),
|
)
|
||||||
# Connection tracking
|
|
||||||
'connection': re.compile(
|
|
||||||
r'(?P<src_ip>[\d.]+):(?P<src_port>\d+)->(?P<dst_ip>[\d.]+):(?P<dst_port>\d+).*'
|
|
||||||
r'proto (?P<proto>\w+).*'
|
|
||||||
r'(?:packets: (?P<packets>\d+))?.*'
|
|
||||||
r'(?:bytes: (?P<bytes>\d+))?'
|
|
||||||
),
|
|
||||||
}
|
|
||||||
|
|
||||||
def connect_db(self):
|
def connect_db(self):
|
||||||
"""Connessione al database PostgreSQL"""
|
"""Connessione al database PostgreSQL"""
|
||||||
@ -65,9 +57,13 @@ class SyslogParser:
|
|||||||
"""
|
"""
|
||||||
Analizza una singola riga di log MikroTik
|
Analizza una singola riga di log MikroTik
|
||||||
Returns: Dict con dati parsati o None se non parsabile
|
Returns: Dict con dati parsati o None se non parsabile
|
||||||
|
|
||||||
|
Formato reale:
|
||||||
|
Nov 17 16:52:16 FIBRA forward: in:... proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280
|
||||||
|
Nov 17 16:52:16 FIBRA detected-ddos forward: ... proto TCP (SYN), 82.62.84.108:43863->185.203.26.34:8472, len 210
|
||||||
"""
|
"""
|
||||||
# Estrai timestamp, hostname, tag e messaggio
|
# Estrai timestamp, hostname, messaggio
|
||||||
# Formato: Jan 15 10:30:45 router1 firewall,info: drop src-address=...
|
# Formato: Nov 17 16:52:16 FIBRA forward: ...
|
||||||
parts = line.split(None, 4)
|
parts = line.split(None, 4)
|
||||||
if len(parts) < 5:
|
if len(parts) < 5:
|
||||||
return None
|
return None
|
||||||
@ -84,20 +80,25 @@ class SyslogParser:
|
|||||||
except ValueError:
|
except ValueError:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
# Prova pattern firewall
|
# Match pattern principale
|
||||||
for pattern_name, pattern in self.patterns.items():
|
match = self.main_pattern.search(message)
|
||||||
match = pattern.search(message)
|
|
||||||
if match:
|
if match:
|
||||||
data = match.groupdict()
|
data = match.groupdict()
|
||||||
|
|
||||||
# Aggiungi metadati
|
# Aggiungi metadati
|
||||||
data['timestamp'] = timestamp
|
data['timestamp'] = timestamp
|
||||||
data['router_name'] = hostname
|
data['router_name'] = hostname
|
||||||
data['log_type'] = pattern_name
|
data['raw_message'] = line.strip()
|
||||||
data['raw_message'] = message.strip()
|
|
||||||
|
# Determina action finale
|
||||||
|
action = data['action']
|
||||||
|
if 'detected-ddos' in action:
|
||||||
|
data['action'] = 'ddos'
|
||||||
|
else:
|
||||||
|
data['action'] = 'forward'
|
||||||
|
|
||||||
# Converti numeri
|
# Converti numeri
|
||||||
for key in ['src_port', 'dst_port', 'len', 'packets', 'bytes']:
|
for key in ['src_port', 'dst_port', 'len']:
|
||||||
if key in data and data[key]:
|
if key in data and data[key]:
|
||||||
data[key] = int(data[key])
|
data[key] = int(data[key])
|
||||||
|
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user