diff --git a/attached_assets/Pasted-head-20-var-log-mikrotik-raw-log-Nov-17-16-52-16-FIBRA-forward-in-sfp-sfpplus1-VS-FTTO-out-sfp-sf-1763400878278_1763400878278.txt b/attached_assets/Pasted-head-20-var-log-mikrotik-raw-log-Nov-17-16-52-16-FIBRA-forward-in-sfp-sfpplus1-VS-FTTO-out-sfp-sf-1763400878278_1763400878278.txt new file mode 100644 index 0000000..fa2ceee --- /dev/null +++ b/attached_assets/Pasted-head-20-var-log-mikrotik-raw-log-Nov-17-16-52-16-FIBRA-forward-in-sfp-sfpplus1-VS-FTTO-out-sfp-sf-1763400878278_1763400878278.txt @@ -0,0 +1,42 @@ +head -20 /var/log/mikrotik/raw.log +Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 +Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 +Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 +Nov 17 16:52:16 FIBRA forward: in:sfp-sfpplus1_VS_FTTO out:sfp-sfpplus2_VS_AS, connection-state:new src-mac c4:ad:34:25:a7:b5, proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:9991->185.203.26.77:53, len 65 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:9991->185.203.26.77:53, len 65 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:43863->185.203.26.34:8472, len 210 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:43863->185.203.26.34:8472, len 210 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56224->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56224->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56225->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56225->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:58268->172.67.143.237:443, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:58268->172.67.143.237:443, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.138:56676->172.67.143.237:80, len 60 +Nov 17 16:52:16 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 185.203.25.233:35832->192.168.25.254:80, len 60 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:56670->185.203.26.34:8472, len 178 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:sfp-sfpplus1_VS_FTTO, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 82.62.84.108:56670->185.203.26.34:8472, len 178 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 72.46.85.161:43970->185.203.24.135:51688, len 44 +Nov 17 16:52:16 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 72.46.85.161:43970->185.203.24.135:51688, len 44 +[root@ids python_ml]# tail -20 /var/log/mikrotik/raw.log +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new src-mac 98:da:c4:75:8c:fb, proto UDP, 10.0.254.170:56065->104.20.23.252:443, len 1278 +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new src-mac 98:da:c4:75:8c:fb, proto UDP, 10.0.254.170:56065->104.20.23.252:443, len 1278 +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new,snat src-mac 98:da:c4:75:8c:fb, proto UDP, 10.0.254.170:56065->104.20.23.252:443, NAT (10.0.254.170:56065->185.203.27.253:56065)->104.20.23.252:443, len 1278 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 126.220.199.81:32730->185.203.25.204:53, len 82 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 126.220.199.81:32730->185.203.25.204:53, len 82 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 160.202.129.17:43994->185.203.24.15:56929, len 44 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 160.202.129.17:43994->185.203.24.15:56929, len 44 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 95.216.123.229:4653->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 95.216.123.229:4653->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:28065->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:28065->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 168.227.31.21:59518->185.203.25.204:53, len 63 +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 10.0.254.242:47946->3.223.194.130:443, len 60 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 168.227.31.21:59518->185.203.25.204:53, len 63 +Nov 17 18:34:26 FIBRA forward: in: out:sfp-sfpplus2_VS_AS, connection-state:new proto TCP (SYN), 10.0.254.242:47946->3.223.194.130:443, len 60 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:3117->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:3117->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:30733->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:, connection-state:new src-mac 18:fd:74:7c:aa:85, proto UDP, 198.251.84.34:30733->185.203.26.77:53, len 65 +Nov 17 18:34:26 FIBRA detected-ddos forward: in:sfp-sfpplus2_VS_AS out:VLAN53_PPOE_DATACENTER, connection-state:new src-mac 18:fd:74:7c:aa:85, proto TCP (SYN), 35.203.211.209:50481->185.203.24.138:27482, len 44 \ No newline at end of file diff --git a/python_ml/syslog_parser.py b/python_ml/syslog_parser.py index 8298562..c1a7f51 100644 --- a/python_ml/syslog_parser.py +++ b/python_ml/syslog_parser.py @@ -23,25 +23,17 @@ class SyslogParser: self.conn = None self.cursor = None - # Pattern regex per parsare log MikroTik - # Formato: timestamp hostname tag: message - self.patterns = { - # Firewall connection - 'firewall': re.compile( - r'(?Paccept|drop|reject).*' - r'src-address=(?P[\d.]+):(?P\d+).*' - r'dst-address=(?P[\d.]+):(?P\d+).*' - r'proto=(?P\w+).*' - r'(?:len=(?P\d+))?' - ), - # Connection tracking - 'connection': re.compile( - r'(?P[\d.]+):(?P\d+)->(?P[\d.]+):(?P\d+).*' - r'proto (?P\w+).*' - r'(?:packets: (?P\d+))?.*' - r'(?:bytes: (?P\d+))?' - ), - } + # Pattern regex per parsare log MikroTik (formato reale) + # Esempio: Nov 17 16:52:16 FIBRA forward: ... proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 + # Esempio: Nov 17 16:52:16 FIBRA detected-ddos forward: ... proto TCP (SYN), 82.62.84.108:43863->185.203.26.34:8472, len 210 + + self.main_pattern = re.compile( + r'(?Pforward|detected-ddos forward):.*?' + r'proto (?PUDP|TCP|ICMP)(?:\s+\((?P[^)]+)\))?.*?' + r'(?P[\d.]+):(?P\d+)->(?P[\d.]+):(?P\d+).*?' + r'len (?P\d+)', + re.IGNORECASE + ) def connect_db(self): """Connessione al database PostgreSQL""" @@ -65,9 +57,13 @@ class SyslogParser: """ Analizza una singola riga di log MikroTik Returns: Dict con dati parsati o None se non parsabile + + Formato reale: + Nov 17 16:52:16 FIBRA forward: in:... proto UDP, 185.203.26.34:55841->192.178.203.94:443, len 1280 + Nov 17 16:52:16 FIBRA detected-ddos forward: ... proto TCP (SYN), 82.62.84.108:43863->185.203.26.34:8472, len 210 """ - # Estrai timestamp, hostname, tag e messaggio - # Formato: Jan 15 10:30:45 router1 firewall,info: drop src-address=... + # Estrai timestamp, hostname, messaggio + # Formato: Nov 17 16:52:16 FIBRA forward: ... parts = line.split(None, 4) if len(parts) < 5: return None @@ -84,24 +80,29 @@ class SyslogParser: except ValueError: return None - # Prova pattern firewall - for pattern_name, pattern in self.patterns.items(): - match = pattern.search(message) - if match: - data = match.groupdict() - - # Aggiungi metadati - data['timestamp'] = timestamp - data['router_name'] = hostname - data['log_type'] = pattern_name - data['raw_message'] = message.strip() - - # Converti numeri - for key in ['src_port', 'dst_port', 'len', 'packets', 'bytes']: - if key in data and data[key]: - data[key] = int(data[key]) - - return data + # Match pattern principale + match = self.main_pattern.search(message) + if match: + data = match.groupdict() + + # Aggiungi metadati + data['timestamp'] = timestamp + data['router_name'] = hostname + data['raw_message'] = line.strip() + + # Determina action finale + action = data['action'] + if 'detected-ddos' in action: + data['action'] = 'ddos' + else: + data['action'] = 'forward' + + # Converti numeri + for key in ['src_port', 'dst_port', 'len']: + if key in data and data[key]: + data[key] = int(data[key]) + + return data return None