837f7d4c08
Corrects the key names used to retrieve detection results in `compare_models.py` to match the output format of the hybrid detector. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: 600ade79-ad9b-4993-b968-e6466b703598 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/RJGlbTt
266 lines
8.5 KiB
Python
266 lines
8.5 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
IDS Model Comparison Script
|
|
Confronta detection del vecchio modello (1.0.0) con il nuovo Hybrid Detector (2.0.0)
|
|
"""
|
|
|
|
import psycopg2
|
|
from psycopg2.extras import RealDictCursor
|
|
import pandas as pd
|
|
from datetime import datetime
|
|
import os
|
|
from dotenv import load_dotenv
|
|
from ml_hybrid_detector import MLHybridDetector
|
|
from ml_analyzer import MLAnalyzer
|
|
|
|
load_dotenv()
|
|
|
|
|
|
def get_db_connection():
|
|
"""Connect to PostgreSQL database"""
|
|
return psycopg2.connect(
|
|
host=os.getenv('PGHOST', 'localhost'),
|
|
port=os.getenv('PGPORT', 5432),
|
|
database=os.getenv('PGDATABASE', 'ids'),
|
|
user=os.getenv('PGUSER', 'postgres'),
|
|
password=os.getenv('PGPASSWORD')
|
|
)
|
|
|
|
|
|
def load_old_detections(limit=100):
|
|
"""
|
|
Carica le detection esistenti dal database
|
|
(non filtriamo per model_version perché la colonna non esiste)
|
|
"""
|
|
print("\n[1] Caricamento detection esistenti dal database...")
|
|
|
|
conn = get_db_connection()
|
|
cursor = conn.cursor(cursor_factory=RealDictCursor)
|
|
|
|
query = """
|
|
SELECT
|
|
d.id,
|
|
d.source_ip,
|
|
d.risk_score,
|
|
d.anomaly_type,
|
|
d.log_count,
|
|
d.last_seen,
|
|
d.blocked,
|
|
d.detected_at
|
|
FROM detections d
|
|
ORDER BY d.risk_score DESC
|
|
LIMIT %s
|
|
"""
|
|
|
|
cursor.execute(query, (limit,))
|
|
detections = cursor.fetchall()
|
|
cursor.close()
|
|
conn.close()
|
|
|
|
print(f" Trovate {len(detections)} detection nel database")
|
|
|
|
return detections
|
|
|
|
|
|
def get_network_logs_for_ip(ip_address, days=7):
|
|
"""
|
|
Recupera i log di rete per un IP specifico (ultimi N giorni)
|
|
"""
|
|
conn = get_db_connection()
|
|
cursor = conn.cursor(cursor_factory=RealDictCursor)
|
|
|
|
query = """
|
|
SELECT
|
|
timestamp,
|
|
source_ip,
|
|
destination_ip as dest_ip,
|
|
destination_port as dest_port,
|
|
protocol,
|
|
packet_length,
|
|
action
|
|
FROM network_logs
|
|
WHERE source_ip = %s
|
|
AND timestamp > NOW() - INTERVAL '1 day' * %s
|
|
ORDER BY timestamp DESC
|
|
LIMIT 10000
|
|
"""
|
|
|
|
cursor.execute(query, (ip_address, days))
|
|
rows = cursor.fetchall()
|
|
cursor.close()
|
|
conn.close()
|
|
|
|
return rows
|
|
|
|
|
|
def reanalyze_with_hybrid(detector, ip_address, old_detection):
|
|
"""
|
|
Rianalizza un IP con il nuovo Hybrid Detector
|
|
"""
|
|
# Recupera log per questo IP
|
|
logs = get_network_logs_for_ip(ip_address, days=7)
|
|
|
|
if not logs:
|
|
return None
|
|
|
|
df = pd.DataFrame(logs)
|
|
|
|
# Il metodo detect() fa già l'extraction delle feature internamente
|
|
# Passiamo direttamente i log grezzi
|
|
result = detector.detect(df, mode='all') # mode='all' per vedere tutti i risultati
|
|
|
|
if not result or len(result) == 0:
|
|
return None
|
|
|
|
# Il detector raggruppa per source_ip, quindi dovrebbe esserci 1 risultato
|
|
new_detection = result[0]
|
|
|
|
# Confronto
|
|
new_score = new_detection.get('risk_score', 0)
|
|
new_type = new_detection.get('anomaly_type', 'unknown')
|
|
new_confidence = new_detection.get('confidence_level', 'unknown')
|
|
|
|
# Determina se è anomalia (score >= 80 = critical threshold)
|
|
new_is_anomaly = new_score >= 80
|
|
|
|
comparison = {
|
|
'ip_address': ip_address,
|
|
'logs_count': len(logs),
|
|
|
|
# Detection corrente nel DB
|
|
'old_score': float(old_detection['risk_score']),
|
|
'old_anomaly_type': old_detection['anomaly_type'],
|
|
'old_blocked': old_detection['blocked'],
|
|
|
|
# Nuovo modello Hybrid (rianalisi)
|
|
'new_score': new_score,
|
|
'new_anomaly_type': new_type,
|
|
'new_confidence': new_confidence,
|
|
'new_is_anomaly': new_is_anomaly,
|
|
|
|
# Delta
|
|
'score_delta': new_score - float(old_detection['risk_score']),
|
|
'type_changed': old_detection['anomaly_type'] != new_type,
|
|
}
|
|
|
|
return comparison
|
|
|
|
|
|
def main():
|
|
print("\n" + "="*80)
|
|
print(" IDS MODEL COMPARISON - DB Current vs Hybrid Detector v2.0.0")
|
|
print("="*80)
|
|
|
|
# Carica detection esistenti
|
|
old_detections = load_old_detections(limit=50)
|
|
|
|
if not old_detections:
|
|
print("\n❌ Nessuna detection trovata nel database!")
|
|
return
|
|
|
|
# Carica nuovo modello Hybrid
|
|
print("\n[2] Caricamento nuovo Hybrid Detector (v2.0.0)...")
|
|
detector = MLHybridDetector(model_dir="models")
|
|
|
|
if not detector.load_models():
|
|
print("\n❌ Modelli Hybrid non trovati! Esegui prima il training:")
|
|
print(" sudo /opt/ids/deployment/run_ml_training.sh")
|
|
return
|
|
|
|
print(f" ✅ Hybrid Detector caricato (18 feature selezionate)")
|
|
|
|
# Rianalizza ogni IP con nuovo modello
|
|
print(f"\n[3] Rianalisi di {len(old_detections)} IP con nuovo modello Hybrid...")
|
|
print(" (Questo può richiedere alcuni minuti...)")
|
|
|
|
comparisons = []
|
|
|
|
for i, old_det in enumerate(old_detections):
|
|
ip = old_det['source_ip']
|
|
|
|
print(f"\n [{i+1}/{len(old_detections)}] Analisi IP: {ip}")
|
|
print(f" Current: score={float(old_det['risk_score']):.1f}, type={old_det['anomaly_type']}, blocked={old_det['blocked']}")
|
|
|
|
comparison = reanalyze_with_hybrid(detector, ip, old_det)
|
|
|
|
if comparison:
|
|
comparisons.append(comparison)
|
|
print(f" Hybrid: score={comparison['new_score']:.1f}, type={comparison['new_anomaly_type']}, confidence={comparison['new_confidence']}")
|
|
print(f" Δ: {comparison['score_delta']:+.1f} score")
|
|
else:
|
|
print(f" ⚠ Nessun log recente trovato per questo IP")
|
|
|
|
# Riepilogo
|
|
print("\n" + "="*80)
|
|
print(" RISULTATI CONFRONTO")
|
|
print("="*80)
|
|
|
|
if not comparisons:
|
|
print("\n❌ Nessun IP rianalizzato (log non disponibili)")
|
|
return
|
|
|
|
df_comp = pd.DataFrame(comparisons)
|
|
|
|
# Statistiche
|
|
print(f"\nIP rianalizzati: {len(comparisons)}/{len(old_detections)}")
|
|
print(f"\nScore medio:")
|
|
print(f" Detection correnti: {df_comp['old_score'].mean():.1f}")
|
|
print(f" Hybrid Detector: {df_comp['new_score'].mean():.1f}")
|
|
print(f" Delta medio: {df_comp['score_delta'].mean():+.1f}")
|
|
|
|
# False Positives (DB aveva score alto, Hybrid dice normale)
|
|
false_positives = df_comp[
|
|
(df_comp['old_score'] >= 80) &
|
|
(~df_comp['new_is_anomaly'])
|
|
]
|
|
|
|
print(f"\n🎯 Possibili False Positives ridotti: {len(false_positives)}")
|
|
if len(false_positives) > 0:
|
|
print("\n IP con score alto nel DB ma ritenuti normali dal Hybrid Detector:")
|
|
for _, row in false_positives.iterrows():
|
|
print(f" • {row['ip_address']} (DB={row['old_score']:.0f}, Hybrid={row['new_score']:.0f})")
|
|
|
|
# True Positives confermati
|
|
true_positives = df_comp[
|
|
(df_comp['old_score'] >= 80) &
|
|
(df_comp['new_is_anomaly'])
|
|
]
|
|
|
|
print(f"\n✅ Anomalie confermate da Hybrid Detector: {len(true_positives)}")
|
|
|
|
# Confidence breakdown (solo nuovo modello)
|
|
if 'new_confidence' in df_comp.columns:
|
|
print(f"\n📊 Confidence Level distribuzione (Hybrid Detector):")
|
|
conf_counts = df_comp['new_confidence'].value_counts()
|
|
for conf, count in conf_counts.items():
|
|
print(f" • {conf}: {count} IP")
|
|
|
|
# Type changes
|
|
type_changes = df_comp[df_comp['type_changed']]
|
|
print(f"\n🔄 IP con cambio tipo anomalia: {len(type_changes)}")
|
|
|
|
# Top 10 maggiori riduzioni score
|
|
print(f"\n📉 Top 10 riduzioni score (possibili FP corretti):")
|
|
top_reductions = df_comp.nsmallest(10, 'score_delta')
|
|
for i, row in enumerate(top_reductions.itertuples(), 1):
|
|
print(f" {i}. {row.ip_address}: {row.old_score:.0f} → {row.new_score:.0f} ({row.score_delta:+.0f})")
|
|
|
|
# Top 10 maggiori aumenti score
|
|
print(f"\n📈 Top 10 aumenti score (nuove anomalie scoperte):")
|
|
top_increases = df_comp.nlargest(10, 'score_delta')
|
|
for i, row in enumerate(top_increases.itertuples(), 1):
|
|
print(f" {i}. {row.ip_address}: {row.old_score:.0f} → {row.new_score:.0f} ({row.score_delta:+.0f})")
|
|
|
|
# Salva CSV per analisi dettagliata
|
|
output_file = f"model_comparison_{datetime.now().strftime('%Y%m%d_%H%M%S')}.csv"
|
|
df_comp.to_csv(output_file, index=False)
|
|
print(f"\n💾 Risultati completi salvati in: {output_file}")
|
|
|
|
print("\n" + "="*80)
|
|
print("✅ Confronto completato!")
|
|
print("="*80 + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|