#!/usr/bin/env python3 """ IDS Model Comparison Script Confronta detection del vecchio modello (1.0.0) con il nuovo Hybrid Detector (2.0.0) """ import psycopg2 from psycopg2.extras import RealDictCursor import pandas as pd from datetime import datetime import os from dotenv import load_dotenv from ml_hybrid_detector import MLHybridDetector from ml_analyzer import MLAnalyzer load_dotenv() def get_db_connection(): """Connect to PostgreSQL database""" return psycopg2.connect( host=os.getenv('PGHOST', 'localhost'), port=os.getenv('PGPORT', 5432), database=os.getenv('PGDATABASE', 'ids'), user=os.getenv('PGUSER', 'postgres'), password=os.getenv('PGPASSWORD') ) def load_old_detections(limit=100): """ Carica le detection esistenti dal database (non filtriamo per model_version perché la colonna non esiste) """ print("\n[1] Caricamento detection esistenti dal database...") conn = get_db_connection() cursor = conn.cursor(cursor_factory=RealDictCursor) query = """ SELECT d.id, d.source_ip, d.risk_score, d.anomaly_type, d.log_count, d.last_seen, d.blocked, d.detected_at FROM detections d ORDER BY d.risk_score DESC LIMIT %s """ cursor.execute(query, (limit,)) detections = cursor.fetchall() cursor.close() conn.close() print(f" Trovate {len(detections)} detection nel database") return detections def get_network_logs_for_ip(ip_address, days=7): """ Recupera i log di rete per un IP specifico (ultimi N giorni) """ conn = get_db_connection() cursor = conn.cursor(cursor_factory=RealDictCursor) query = """ SELECT timestamp, source_ip, destination_ip as dest_ip, destination_port as dest_port, protocol, packet_length, action FROM network_logs WHERE source_ip = %s AND timestamp > NOW() - INTERVAL '1 day' * %s ORDER BY timestamp DESC LIMIT 10000 """ cursor.execute(query, (ip_address, days)) rows = cursor.fetchall() cursor.close() conn.close() return rows def reanalyze_with_hybrid(detector, ip_address, old_detection): """ Rianalizza un IP con il nuovo Hybrid Detector """ # Recupera log per questo IP logs = get_network_logs_for_ip(ip_address, days=7) if not logs: return None df = pd.DataFrame(logs) # Feature extraction (come nel detector) features_df = detector.extract_features(df) if len(features_df) == 0: return None # Prendi le feature dell'IP aggregato ip_features = features_df.iloc[0:1] # Rianalizza con nuovo modello result = detector.detect(ip_features) if not result or len(result) == 0: return None new_detection = result[0] # Confronto comparison = { 'ip_address': ip_address, 'logs_count': len(logs), # Detection corrente nel DB 'old_score': float(old_detection['risk_score']), 'old_anomaly_type': old_detection['anomaly_type'], 'old_blocked': old_detection['blocked'], # Nuovo modello Hybrid (rianalisi) 'new_score': new_detection.get('anomaly_score', 0), 'new_anomaly_type': new_detection.get('anomaly_type', 'unknown'), 'new_confidence': new_detection.get('confidence', 'unknown'), 'new_is_anomaly': new_detection.get('is_anomaly', False), # Delta 'score_delta': new_detection.get('anomaly_score', 0) - float(old_detection['risk_score']), 'type_changed': old_detection['anomaly_type'] != new_detection.get('anomaly_type', 'unknown'), } return comparison def main(): print("\n" + "="*80) print(" IDS MODEL COMPARISON - DB Current vs Hybrid Detector v2.0.0") print("="*80) # Carica detection esistenti old_detections = load_old_detections(limit=50) if not old_detections: print("\n❌ Nessuna detection trovata nel database!") return # Carica nuovo modello Hybrid print("\n[2] Caricamento nuovo Hybrid Detector (v2.0.0)...") detector = MLHybridDetector(model_dir="models") if not detector.load_models(): print("\n❌ Modelli Hybrid non trovati! Esegui prima il training:") print(" sudo /opt/ids/deployment/run_ml_training.sh") return print(f" ✅ Hybrid Detector caricato (18 feature selezionate)") # Rianalizza ogni IP con nuovo modello print(f"\n[3] Rianalisi di {len(old_detections)} IP con nuovo modello Hybrid...") print(" (Questo può richiedere alcuni minuti...)") comparisons = [] for i, old_det in enumerate(old_detections): ip = old_det['source_ip'] print(f"\n [{i+1}/{len(old_detections)}] Analisi IP: {ip}") print(f" Current: score={float(old_det['risk_score']):.1f}, type={old_det['anomaly_type']}, blocked={old_det['blocked']}") comparison = reanalyze_with_hybrid(detector, ip, old_det) if comparison: comparisons.append(comparison) print(f" Hybrid: score={comparison['new_score']:.1f}, type={comparison['new_anomaly_type']}, confidence={comparison['new_confidence']}") print(f" Δ: {comparison['score_delta']:+.1f} score") else: print(f" ⚠ Nessun log recente trovato per questo IP") # Riepilogo print("\n" + "="*80) print(" RISULTATI CONFRONTO") print("="*80) if not comparisons: print("\n❌ Nessun IP rianalizzato (log non disponibili)") return df_comp = pd.DataFrame(comparisons) # Statistiche print(f"\nIP rianalizzati: {len(comparisons)}/{len(old_detections)}") print(f"\nScore medio:") print(f" Detection correnti: {df_comp['old_score'].mean():.1f}") print(f" Hybrid Detector: {df_comp['new_score'].mean():.1f}") print(f" Delta medio: {df_comp['score_delta'].mean():+.1f}") # False Positives (DB aveva score alto, Hybrid dice normale) false_positives = df_comp[ (df_comp['old_score'] >= 80) & (~df_comp['new_is_anomaly']) ] print(f"\n🎯 Possibili False Positives ridotti: {len(false_positives)}") if len(false_positives) > 0: print("\n IP con score alto nel DB ma ritenuti normali dal Hybrid Detector:") for _, row in false_positives.iterrows(): print(f" • {row['ip_address']} (DB={row['old_score']:.0f}, Hybrid={row['new_score']:.0f})") # True Positives confermati true_positives = df_comp[ (df_comp['old_score'] >= 80) & (df_comp['new_is_anomaly']) ] print(f"\n✅ Anomalie confermate da Hybrid Detector: {len(true_positives)}") # Confidence breakdown (solo nuovo modello) if 'new_confidence' in df_comp.columns: print(f"\n📊 Confidence Level distribuzione (Hybrid Detector):") conf_counts = df_comp['new_confidence'].value_counts() for conf, count in conf_counts.items(): print(f" • {conf}: {count} IP") # Type changes type_changes = df_comp[df_comp['type_changed']] print(f"\n🔄 IP con cambio tipo anomalia: {len(type_changes)}") # Top 10 maggiori riduzioni score print(f"\n📉 Top 10 riduzioni score (possibili FP corretti):") top_reductions = df_comp.nsmallest(10, 'score_delta') for i, row in enumerate(top_reductions.itertuples(), 1): print(f" {i}. {row.ip_address}: {row.old_score:.0f} → {row.new_score:.0f} ({row.score_delta:+.0f})") # Top 10 maggiori aumenti score print(f"\n📈 Top 10 aumenti score (nuove anomalie scoperte):") top_increases = df_comp.nlargest(10, 'score_delta') for i, row in enumerate(top_increases.itertuples(), 1): print(f" {i}. {row.ip_address}: {row.old_score:.0f} → {row.new_score:.0f} ({row.score_delta:+.0f})") # Salva CSV per analisi dettagliata output_file = f"model_comparison_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv" df_comp.to_csv(output_file, index=False) print(f"\n💾 Risultati completi salvati in: {output_file}") print("\n" + "="*80) print("✅ Confronto completato!") print("="*80 + "\n") if __name__ == "__main__": main()