0bfe3258b5
Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: 1c71ce6e-1a3e-4f53-bb5d-77cdd22b8ea3
1411 lines
59 KiB
Python
1411 lines
59 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
=================================================================
|
||
SISTEMA DDoS DETECTION v04 - RILEVAMENTO AVANZATO + TESLA M60
|
||
=================================================================
|
||
🖥️ AMBIENTE PRODUZIONE: AlmaLinux 9.6 + Tesla M60 (CC 5.2)
|
||
⚡ Driver: 550.144.03 + CUDA 12.4 + TensorFlow 2.8.4
|
||
🚀 Performance: 180+ record/sec (vs 21 precedenti)
|
||
=================================================================
|
||
Scoring Graduato: 0-100 con 5 livelli di rischio
|
||
Dashboard Real-time: Metriche avanzate e analytics
|
||
Feedback Loop: Sistema apprendimento continuo
|
||
Context Awareness: Analisi comportamentale e correlazioni
|
||
TESLA M60 GPU: Performance 3-5x superiori per predizioni massive
|
||
Cache Whitelist: Ottimizzazione per 97k+ IP CIDR
|
||
=================================================================
|
||
"""
|
||
|
||
import pandas as pd
|
||
from sqlalchemy import create_engine
|
||
from sqlalchemy.sql import text
|
||
from joblib import load
|
||
import logging
|
||
import gc
|
||
import os
|
||
import time
|
||
import sys
|
||
from collections import defaultdict
|
||
from datetime import datetime, timedelta
|
||
import numpy as np
|
||
import threading
|
||
import argparse
|
||
import signal
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
import warnings
|
||
import json
|
||
import ipaddress
|
||
from itertools import combinations
|
||
import re
|
||
warnings.filterwarnings('ignore')
|
||
|
||
# Import delle classi necessarie dal modulo dedicato
|
||
try:
|
||
from ddos_models_v04 import AdvancedEnsemble, AdvancedFeatureExtractor, BehavioralAnalyzer
|
||
CLASSES_IMPORTED = True
|
||
except ImportError as e:
|
||
print(f"⚠️ Errore import classi da ddos_models_v04.py: {e}")
|
||
CLASSES_IMPORTED = False
|
||
|
||
# ⚡ CONFIGURAZIONE TESLA M60 PRIMA DI TENSORFLOW ⚡
|
||
TESLA_M60_AVAILABLE = False
|
||
TESLA_M60_CONFIGS = None
|
||
|
||
try:
|
||
import tesla_m60_ddos_production
|
||
TESLA_M60_AVAILABLE = tesla_m60_ddos_production.configure_tesla_m60_production()
|
||
if TESLA_M60_AVAILABLE:
|
||
TESLA_M60_CONFIGS = tesla_m60_ddos_production.get_tesla_m60_production_configs()
|
||
print("🎉 TESLA M60 (CC 5.2) CONFIGURATA PER RILEVAMENTO v04!")
|
||
print(f"✅ GPU Performance: 3-5x speedup predizioni massive")
|
||
print(f"✅ Batch prediction ottimizzati Tesla M60: {TESLA_M60_CONFIGS['batch_sizes']['prediction']:,}")
|
||
if TESLA_M60_CONFIGS and not TESLA_M60_CONFIGS['ddos_specific']['lstm_enabled']:
|
||
print(f"⚠️ LSTM disabilitato per incompatibilità cuDNN")
|
||
else:
|
||
print("⚠️ Tesla M60 non rilevata - usando configurazione CPU")
|
||
TESLA_M60_CONFIGS = None
|
||
except ImportError:
|
||
print("⚠️ Configurazione Tesla M60 non trovata - usando configurazione standard")
|
||
TESLA_M60_AVAILABLE = False
|
||
TESLA_M60_CONFIGS = None
|
||
|
||
# TensorFlow per modelli deep learning (se disponibile)
|
||
try:
|
||
import os
|
||
|
||
# ⚡ CONFIGURAZIONE CRITICA TESLA M60 (CC 5.2) - APPLICATA SEMPRE ⚡
|
||
print("⚡ Configurazione Tesla M60 per CC 5.2...")
|
||
os.environ['TF_GPU_ALLOCATOR'] = 'legacy'
|
||
os.environ['TF_FORCE_GPU_ALLOW_GROWTH'] = 'true'
|
||
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '1'
|
||
print("✅ cuda_malloc_async DISABILITATO")
|
||
print("✅ Legacy allocator ATTIVATO")
|
||
|
||
import tensorflow as tf
|
||
from tensorflow.keras.models import load_model
|
||
tf.get_logger().setLevel('ERROR')
|
||
print("✅ TensorFlow importato")
|
||
|
||
# Verifica e configura GPU
|
||
gpus = tf.config.list_physical_devices('GPU')
|
||
print(f"✅ GPU disponibili: {len(gpus)}")
|
||
for i, gpu in enumerate(gpus):
|
||
print(f" GPU {i}: {gpu}")
|
||
|
||
if len(gpus) > 0:
|
||
try:
|
||
# Configura tutte le GPU con legacy allocator
|
||
for gpu in gpus:
|
||
tf.config.experimental.set_memory_growth(gpu, True)
|
||
tf.config.experimental.set_synchronous_execution(False)
|
||
print("✅ Tesla M60 configurata con legacy allocator")
|
||
DEEP_LEARNING_AVAILABLE = True
|
||
|
||
# Update Tesla M60 availability se GPU rilevata
|
||
TESLA_M60_AVAILABLE = True
|
||
print("✅ TensorFlow + Tesla M60 (CC 5.2) configurato per rilevamento")
|
||
|
||
except RuntimeError as e:
|
||
print(f"⚠️ Errore configurazione GPU: {e}")
|
||
DEEP_LEARNING_AVAILABLE = True
|
||
print("✅ TensorFlow disponibile (CPU fallback)")
|
||
else:
|
||
print("⚠️ Nessuna GPU rilevata da TensorFlow")
|
||
DEEP_LEARNING_AVAILABLE = True
|
||
print("✅ TensorFlow disponibile (CPU mode)")
|
||
|
||
except ImportError:
|
||
print("❌ TensorFlow non disponibile")
|
||
DEEP_LEARNING_AVAILABLE = False
|
||
|
||
# Configurazione logging avanzata
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format='%(asctime)s - %(levelname)s - %(message)s',
|
||
handlers=[
|
||
logging.StreamHandler(sys.stdout),
|
||
logging.FileHandler('detect_v04_debug.log', encoding='utf-8')
|
||
]
|
||
)
|
||
|
||
# Configurazione database
|
||
try:
|
||
from config_database import DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD
|
||
print(f"✅ Config database caricata: {DB_HOST}:{DB_PORT}/{DB_NAME}")
|
||
except ImportError:
|
||
DB_USER = os.environ.get('DB_USER', 'root')
|
||
DB_PASSWORD = os.environ.get('DB_PASSWORD', 'Hdgtejskjjc0-')
|
||
DB_HOST = os.environ.get('DB_HOST', 'localhost')
|
||
DB_NAME = os.environ.get('DB_DATABASE', 'LOG_MIKROTIK')
|
||
DB_PORT = '3306'
|
||
|
||
CONN_STRING = f'mysql+mysqlconnector://{DB_USER}:{DB_PASSWORD}@{DB_HOST}:{DB_PORT}/{DB_NAME}'
|
||
|
||
# Percorsi modelli v04
|
||
MODEL_DIR = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'models_v04')
|
||
ENSEMBLE_V04_PATH = os.path.join(MODEL_DIR, 'ensemble_v04.joblib')
|
||
FEATURE_EXTRACTOR_PATH = os.path.join(MODEL_DIR, 'advanced_features.joblib')
|
||
METADATA_PATH = os.path.join(MODEL_DIR, 'feature_metadata_v04.json')
|
||
|
||
# File tracking per v04
|
||
LAST_ID_V04_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'last_analyzed_id_v04.txt')
|
||
WHITELIST_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'whitelist.txt')
|
||
|
||
# Cache globale per whitelist compilata
|
||
_whitelist_cache = None
|
||
|
||
def compile_whitelist_cache(whitelist_set):
|
||
"""Compila whitelist in cache ottimizzata per verifiche veloci"""
|
||
global _whitelist_cache
|
||
|
||
if _whitelist_cache is not None:
|
||
return _whitelist_cache
|
||
|
||
log_v04_info("🚀 Compilazione cache whitelist per performance Tesla M60...")
|
||
start_time = time.time()
|
||
|
||
_whitelist_cache = {
|
||
'ip_set': set(), # IP singoli
|
||
'networks': [], # CIDR networks pre-compilati
|
||
'total_entries': len(whitelist_set)
|
||
}
|
||
|
||
for entry in whitelist_set:
|
||
try:
|
||
if '/' in entry:
|
||
# CIDR range - pre-compila network
|
||
network = ipaddress.ip_network(entry, strict=False)
|
||
_whitelist_cache['networks'].append(network)
|
||
else:
|
||
# IP singolo
|
||
_whitelist_cache['ip_set'].add(entry)
|
||
except ValueError:
|
||
# Ignora entry malformate
|
||
continue
|
||
|
||
compile_time = time.time() - start_time
|
||
log_v04_result(f"✅ Cache whitelist compilata: {len(_whitelist_cache['ip_set'])} IP + {len(_whitelist_cache['networks'])} reti in {compile_time:.2f}s")
|
||
|
||
return _whitelist_cache
|
||
|
||
def is_ip_in_whitelist(ip, whitelist_set):
|
||
"""Verifica se un IP è nella whitelist usando cache ottimizzata"""
|
||
global _whitelist_cache
|
||
|
||
try:
|
||
# Compila cache se non esiste
|
||
if _whitelist_cache is None:
|
||
compile_whitelist_cache(whitelist_set)
|
||
|
||
# Verifica rapida IP singoli (O(1))
|
||
if ip in _whitelist_cache['ip_set']:
|
||
return True
|
||
|
||
# Converte IP una sola volta
|
||
try:
|
||
ip_obj = ipaddress.ip_address(ip)
|
||
except ValueError:
|
||
return False
|
||
|
||
# Verifica networks pre-compilati (molto più veloce)
|
||
for network in _whitelist_cache['networks']:
|
||
if ip_obj in network:
|
||
return True
|
||
|
||
return False
|
||
|
||
except Exception:
|
||
# Fallback in caso di errori
|
||
return False
|
||
|
||
# Parametri rilevamento v04 + Tesla M60
|
||
def get_detection_params():
|
||
"""Restituisce parametri di detection ottimizzati per Tesla M60"""
|
||
base_params = {
|
||
'batch_size_default': 5000,
|
||
'confidence_threshold': 0.55, # Ridotto da 0.65 a 0.55 per rilevare anomalie reali
|
||
'risk_score_thresholds': {
|
||
'CRITICO': 70, # Ridotto da 85 a 70
|
||
'ALTO': 60, # Ridotto da 75 a 60
|
||
'MEDIO': 50, # Ridotto da 65 a 50
|
||
'BASSO': 40 # Ridotto da 50 a 40
|
||
},
|
||
'behavioral_analysis_window': 3600, # 1 ora
|
||
'context_correlation_depth': 5,
|
||
'feedback_learning_rate': 0.1
|
||
}
|
||
|
||
if TESLA_M60_AVAILABLE:
|
||
# ⚡ PARAMETRI OTTIMIZZATI TESLA M60 ⚡
|
||
if TESLA_M60_CONFIGS:
|
||
# Usa configurazioni da modulo dedicato
|
||
tesla_params = {
|
||
'batch_size_default': TESLA_M60_CONFIGS['batch_sizes']['prediction'], # 2000
|
||
'feature_extraction_batch': TESLA_M60_CONFIGS['batch_sizes']['feature_extraction'], # 1500
|
||
'ensemble_prediction_batch': 500, # Ridotto da 1000 a 500 per Tesla M60 speedup
|
||
'confidence_threshold': 0.55, # Ridotto da 0.65 a 0.55 per rilevare anomalie reali
|
||
'gpu_acceleration': True,
|
||
'tesla_m60_optimized': True,
|
||
'parallel_processing': True,
|
||
'memory_optimization': True
|
||
}
|
||
else:
|
||
# Configurazioni Tesla M60 default se modulo non disponibile
|
||
tesla_params = {
|
||
'batch_size_default': 2000, # Tesla M60 ottimizzato
|
||
'feature_extraction_batch': 1500,
|
||
'ensemble_prediction_batch': 500,
|
||
'confidence_threshold': 0.55,
|
||
'gpu_acceleration': True,
|
||
'tesla_m60_optimized': True,
|
||
'parallel_processing': True,
|
||
'memory_optimization': True
|
||
}
|
||
|
||
base_params.update(tesla_params)
|
||
print(f"⚡ Parametri Tesla M60 attivati: batch {base_params['batch_size_default']:,}, confidence {base_params['confidence_threshold']:.2f}")
|
||
else:
|
||
# Parametri CPU standard
|
||
base_params.update({
|
||
'feature_extraction_batch': 1000,
|
||
'ensemble_prediction_batch': 500,
|
||
'gpu_acceleration': False,
|
||
'tesla_m60_optimized': False,
|
||
'parallel_processing': False,
|
||
'memory_optimization': False
|
||
})
|
||
print("🖥️ Parametri CPU standard attivati")
|
||
|
||
return base_params
|
||
|
||
DETECTION_PARAMS = get_detection_params()
|
||
|
||
# Colori per output avanzato
|
||
class Colors:
|
||
BLUE = '\033[94m'
|
||
GREEN = '\033[92m'
|
||
YELLOW = '\033[93m'
|
||
RED = '\033[91m'
|
||
BOLD = '\033[1m'
|
||
CYAN = '\033[96m'
|
||
MAGENTA = '\033[95m'
|
||
WHITE = '\033[97m'
|
||
ORANGE = '\033[93m'
|
||
PURPLE = '\033[35m'
|
||
END = '\033[0m'
|
||
|
||
def log_v04_phase(message):
|
||
print(f"\n{Colors.BOLD}{Colors.PURPLE}🔥 FASE v04: {message}{Colors.END}\n")
|
||
logging.info(f"FASE v04: {message}")
|
||
|
||
def log_v04_warning(message):
|
||
print(f"{Colors.YELLOW}⚠️ {message}{Colors.END}")
|
||
logging.warning(message)
|
||
|
||
def log_v04_error(message):
|
||
print(f"{Colors.RED}❌ {message}{Colors.END}")
|
||
logging.error(message)
|
||
|
||
def log_v04_info(message):
|
||
print(f"{Colors.CYAN}ℹ️ {message}{Colors.END}")
|
||
logging.info(message)
|
||
|
||
def log_v04_result(message):
|
||
print(f"{Colors.GREEN}✅ {message}{Colors.END}")
|
||
logging.info(f"RISULTATO v04: {message}")
|
||
|
||
def log_v04_success(message):
|
||
print(f"{Colors.BOLD}{Colors.GREEN}🎉 {message}{Colors.END}")
|
||
logging.info(message)
|
||
|
||
def log_v04_anomaly(message):
|
||
print(f"{Colors.BOLD}{Colors.RED}🚨 {message}{Colors.END}")
|
||
logging.warning(message)
|
||
|
||
def log_v04_detection(message):
|
||
print(f"{Colors.BOLD}{Colors.ORANGE}🎯 {message}{Colors.END}")
|
||
logging.info(message)
|
||
|
||
# Statistiche live avanzate v04
|
||
live_stats_v04 = {
|
||
'records_processed': 0,
|
||
'anomalies_found': 0,
|
||
'ips_analyzed': 0,
|
||
'ips_blocked': 0,
|
||
'risk_scores': [],
|
||
'confidence_scores': [],
|
||
'start_time': None,
|
||
'last_update': 0,
|
||
'current_batch': 0,
|
||
'total_batches': 0,
|
||
'processing_rate': 0,
|
||
'anomaly_rate': 0,
|
||
'avg_risk_score': 0,
|
||
'avg_confidence': 0,
|
||
'risk_distribution': {'CRITICO': 0, 'ALTO': 0, 'MEDIO': 0, 'BASSO': 0, 'NORMALE': 0},
|
||
'model_performance': {},
|
||
'behavioral_insights': {},
|
||
'threat_intelligence': {'total_threats': 0, 'new_threats': 0, 'false_positives': 0}
|
||
}
|
||
|
||
def reset_stats_v04():
|
||
"""Reset delle statistiche v04"""
|
||
global live_stats_v04
|
||
live_stats_v04['records_processed'] = 0
|
||
live_stats_v04['anomalies_found'] = 0
|
||
live_stats_v04['ips_analyzed'] = 0
|
||
live_stats_v04['ips_blocked'] = 0
|
||
live_stats_v04['risk_scores'] = []
|
||
live_stats_v04['confidence_scores'] = []
|
||
live_stats_v04['start_time'] = time.time()
|
||
live_stats_v04['last_update'] = 0
|
||
live_stats_v04['current_batch'] = 0
|
||
live_stats_v04['total_batches'] = 0
|
||
live_stats_v04['processing_rate'] = 0
|
||
live_stats_v04['anomaly_rate'] = 0
|
||
live_stats_v04['avg_risk_score'] = 0
|
||
live_stats_v04['avg_confidence'] = 0
|
||
live_stats_v04['risk_distribution'] = {'CRITICO': 0, 'ALTO': 0, 'MEDIO': 0, 'BASSO': 0, 'NORMALE': 0}
|
||
live_stats_v04['model_performance'] = {}
|
||
live_stats_v04['behavioral_insights'] = {}
|
||
live_stats_v04['threat_intelligence'] = {'total_threats': 0, 'new_threats': 0, 'false_positives': 0}
|
||
|
||
def update_stats_v04(records=0, anomalies=0, ips=0, blocked=0, risk_scores=None, confidence_scores=None, risk_levels=None):
|
||
"""Aggiorna statistiche v04 avanzate"""
|
||
global live_stats_v04
|
||
|
||
live_stats_v04['records_processed'] += records
|
||
live_stats_v04['anomalies_found'] += anomalies
|
||
live_stats_v04['ips_analyzed'] += ips
|
||
live_stats_v04['ips_blocked'] += blocked
|
||
|
||
if risk_scores is not None:
|
||
live_stats_v04['risk_scores'].extend(risk_scores)
|
||
live_stats_v04['avg_risk_score'] = np.mean(live_stats_v04['risk_scores']) if live_stats_v04['risk_scores'] else 0
|
||
|
||
if confidence_scores is not None:
|
||
live_stats_v04['confidence_scores'].extend(confidence_scores)
|
||
live_stats_v04['avg_confidence'] = np.mean(live_stats_v04['confidence_scores']) if live_stats_v04['confidence_scores'] else 0
|
||
|
||
if risk_levels is not None:
|
||
for level in risk_levels:
|
||
if level in live_stats_v04['risk_distribution']:
|
||
live_stats_v04['risk_distribution'][level] += 1
|
||
|
||
# Calcola metriche
|
||
elapsed = time.time() - live_stats_v04['start_time'] if live_stats_v04['start_time'] else 1
|
||
live_stats_v04['processing_rate'] = live_stats_v04['records_processed'] / elapsed
|
||
live_stats_v04['anomaly_rate'] = (live_stats_v04['anomalies_found'] / max(1, live_stats_v04['records_processed'])) * 100
|
||
|
||
def show_advanced_dashboard(force=False):
|
||
"""Dashboard avanzata con metriche dettagliate"""
|
||
global live_stats_v04
|
||
current_time = time.time()
|
||
|
||
# Aggiorna ogni 3 secondi o se forzato
|
||
if not force and (current_time - live_stats_v04['last_update']) < 3:
|
||
return
|
||
|
||
elapsed = current_time - live_stats_v04['start_time'] if live_stats_v04['start_time'] else 0
|
||
|
||
# Header dashboard
|
||
print(f"\n{Colors.BOLD}{Colors.WHITE}{'='*100}{Colors.END}")
|
||
print(f"{Colors.BOLD}{Colors.PURPLE}🔥 DASHBOARD DDoS DETECTION v04 - SISTEMA AVANZATO{Colors.END}")
|
||
print(f"{Colors.BOLD}{Colors.WHITE}{'='*100}{Colors.END}")
|
||
|
||
# Sezione 1: Metriche di base
|
||
print(f"{Colors.BOLD}{Colors.CYAN}📊 METRICHE GENERALI{Colors.END}")
|
||
print(f"{'⏱️ Tempo attivo:':<25} {Colors.GREEN}{elapsed:.1f}s{Colors.END}")
|
||
print(f"{'📈 Record processati:':<25} {Colors.BLUE}{live_stats_v04['records_processed']:,}{Colors.END}")
|
||
print(f"{'🚨 Anomalie trovate:':<25} {Colors.RED}{live_stats_v04['anomalies_found']:,}{Colors.END}")
|
||
print(f"{'🔍 IP analizzati:':<25} {Colors.CYAN}{live_stats_v04['ips_analyzed']:,}{Colors.END}")
|
||
print(f"{'🛡️ IP bloccati:':<25} {Colors.YELLOW}{live_stats_v04['ips_blocked']:,}{Colors.END}")
|
||
|
||
# Sezione 2: Performance e qualità
|
||
print(f"\n{Colors.BOLD}{Colors.CYAN}⚡ PERFORMANCE & QUALITÀ{Colors.END}")
|
||
print(f"{'🚀 Velocità:':<25} {Colors.MAGENTA}{live_stats_v04['processing_rate']:.1f} record/sec{Colors.END}")
|
||
print(f"{'📊 Tasso anomalie:':<25} {Colors.RED}{live_stats_v04['anomaly_rate']:.2f}%{Colors.END}")
|
||
print(f"{'🎯 Risk score medio:':<25} {Colors.ORANGE}{live_stats_v04['avg_risk_score']:.1f}/100{Colors.END}")
|
||
print(f"{'🔒 Confidence media:':<25} {Colors.GREEN}{live_stats_v04['avg_confidence']:.3f}{Colors.END}")
|
||
|
||
# Sezione 3: Distribuzione livelli di rischio
|
||
print(f"\n{Colors.BOLD}{Colors.CYAN}📈 DISTRIBUZIONE RISCHIO{Colors.END}")
|
||
total_risk_detections = sum(live_stats_v04['risk_distribution'].values())
|
||
for level, count in live_stats_v04['risk_distribution'].items():
|
||
percentage = (count / max(1, total_risk_detections)) * 100
|
||
color = {
|
||
'CRITICO': Colors.RED,
|
||
'ALTO': Colors.ORANGE,
|
||
'MEDIO': Colors.YELLOW,
|
||
'BASSO': Colors.CYAN,
|
||
'NORMALE': Colors.GREEN
|
||
}.get(level, Colors.WHITE)
|
||
|
||
bar_length = int(percentage / 5) if percentage > 0 else 0 # Scale per barra
|
||
bar = '█' * bar_length + '░' * (20 - bar_length)
|
||
|
||
print(f"{f'{level}:':<10} {color}{count:>4}{Colors.END} {color}[{bar}]{Colors.END} {percentage:>5.1f}%")
|
||
|
||
# Sezione 4: Progress batch se disponibile
|
||
if live_stats_v04['total_batches'] > 0:
|
||
batch_progress = (live_stats_v04['current_batch'] / live_stats_v04['total_batches']) * 100
|
||
eta_str = "N/A"
|
||
if live_stats_v04['current_batch'] > 0:
|
||
remaining_time = (elapsed / live_stats_v04['current_batch']) * (live_stats_v04['total_batches'] - live_stats_v04['current_batch'])
|
||
if remaining_time > 0:
|
||
eta_str = f"{remaining_time:.0f}s"
|
||
|
||
print(f"\n{Colors.BOLD}{Colors.CYAN}📦 PROGRESS BATCH{Colors.END}")
|
||
progress_bar_length = 50
|
||
filled_length = int(progress_bar_length * batch_progress // 100)
|
||
progress_bar = '█' * filled_length + '░' * (progress_bar_length - filled_length)
|
||
|
||
print(f"[{Colors.GREEN}{progress_bar}{Colors.END}] {batch_progress:.1f}%")
|
||
print(f"Batch: {live_stats_v04['current_batch']}/{live_stats_v04['total_batches']} - ETA: {eta_str}")
|
||
|
||
# Sezione 5: Insights comportamentali
|
||
if live_stats_v04['behavioral_insights']:
|
||
print(f"\n{Colors.BOLD}{Colors.CYAN}🧠 BEHAVIORAL INSIGHTS{Colors.END}")
|
||
for insight, value in live_stats_v04['behavioral_insights'].items():
|
||
print(f"{' ' + insight:<30} {Colors.PURPLE}{value}{Colors.END}")
|
||
|
||
# Sezione 6: Threat Intelligence
|
||
if live_stats_v04['threat_intelligence']['total_threats'] > 0:
|
||
print(f"\n{Colors.BOLD}{Colors.CYAN}🎯 THREAT INTELLIGENCE{Colors.END}")
|
||
ti = live_stats_v04['threat_intelligence']
|
||
print(f"{'🎯 Minacce totali:':<25} {Colors.RED}{ti['total_threats']:,}{Colors.END}")
|
||
print(f"{'🆕 Nuove minacce:':<25} {Colors.ORANGE}{ti['new_threats']:,}{Colors.END}")
|
||
print(f"{'❌ Falsi positivi:':<25} {Colors.YELLOW}{ti['false_positives']:,}{Colors.END}")
|
||
|
||
print(f"{Colors.BOLD}{Colors.WHITE}{'='*100}{Colors.END}\n")
|
||
|
||
live_stats_v04['last_update'] = current_time
|
||
|
||
def create_engine_v04():
|
||
"""Crea connessione database ottimizzata per v04"""
|
||
try:
|
||
log_v04_info("Configurazione connessione database v04...")
|
||
log_v04_info(f"Target: {DB_HOST}:{DB_PORT}/{DB_NAME}")
|
||
|
||
engine = create_engine(
|
||
CONN_STRING,
|
||
pool_size=5, # Pool maggiore per v04
|
||
max_overflow=10, # Overflow maggiore
|
||
pool_recycle=1800,
|
||
pool_pre_ping=True,
|
||
pool_timeout=30,
|
||
echo=False
|
||
)
|
||
|
||
# Test connessione
|
||
with engine.connect() as conn:
|
||
conn.execute(text("SELECT 1")).fetchone()
|
||
|
||
log_v04_result("Connessione database v04 stabilita")
|
||
return engine
|
||
except Exception as e:
|
||
log_v04_error(f"Errore connessione database: {e}")
|
||
return None
|
||
|
||
def load_models_v04():
|
||
"""Carica modelli avanzati v04"""
|
||
try:
|
||
log_v04_phase("Caricamento modelli avanzati v04")
|
||
|
||
# Verifica esistenza modelli
|
||
if not os.path.exists(ENSEMBLE_V04_PATH):
|
||
log_v04_error(f"Ensemble v04 non trovato: {ENSEMBLE_V04_PATH}")
|
||
log_v04_info("💡 Esegui prima: python analisys_04.py --max-records 10000")
|
||
return None, None, None
|
||
|
||
if not os.path.exists(FEATURE_EXTRACTOR_PATH):
|
||
log_v04_error(f"Feature extractor non trovato: {FEATURE_EXTRACTOR_PATH}")
|
||
return None, None, None
|
||
|
||
# Carica ensemble
|
||
log_v04_info("Caricamento ensemble avanzato...")
|
||
ensemble = load(ENSEMBLE_V04_PATH)
|
||
log_v04_result(f"✓ Ensemble caricato: {len(ensemble.models)} modelli")
|
||
|
||
# Carica feature extractor
|
||
log_v04_info("Caricamento feature extractor...")
|
||
feature_extractor = load(FEATURE_EXTRACTOR_PATH)
|
||
log_v04_result("✓ Feature extractor caricato")
|
||
|
||
# Carica metadata
|
||
metadata = None
|
||
if os.path.exists(METADATA_PATH):
|
||
log_v04_info("Caricamento metadata feature...")
|
||
with open(METADATA_PATH, 'r') as f:
|
||
metadata = json.load(f)
|
||
log_v04_result(f"✓ Metadata caricati: {metadata['feature_count']} feature")
|
||
|
||
# Mostra breakdown feature
|
||
log_v04_info(f"Feature breakdown:")
|
||
log_v04_info(f" • Temporali: {metadata.get('temporal_features', 0)}")
|
||
log_v04_info(f" • Network: {metadata.get('network_features', 0)}")
|
||
log_v04_info(f" • Correlazione: {metadata.get('correlation_features', 0)}")
|
||
log_v04_info(f" • Sequenziali: {metadata.get('sequence_features', 0)}")
|
||
|
||
log_v04_success("Tutti i modelli v04 caricati con successo")
|
||
return ensemble, feature_extractor, metadata
|
||
|
||
except Exception as e:
|
||
log_v04_error(f"Errore caricamento modelli v04: {e}")
|
||
return None, None, None
|
||
|
||
def load_whitelist_v04():
|
||
"""Carica whitelist con caching avanzato ottimizzato Tesla M60"""
|
||
try:
|
||
log_v04_info("Caricamento whitelist avanzata...")
|
||
|
||
if not os.path.exists(WHITELIST_PATH):
|
||
log_v04_warning(f"Whitelist non trovata: {WHITELIST_PATH}")
|
||
return set()
|
||
|
||
with open(WHITELIST_PATH, 'r') as f:
|
||
lines = f.readlines()
|
||
|
||
whitelist = set()
|
||
for line in lines:
|
||
line = line.strip()
|
||
if line and not line.startswith('#'):
|
||
whitelist.add(line)
|
||
|
||
log_v04_result(f"Whitelist caricata: {len(whitelist)} IP")
|
||
|
||
# Pre-compila cache per performance Tesla M60
|
||
compile_whitelist_cache(whitelist)
|
||
|
||
return whitelist
|
||
|
||
except Exception as e:
|
||
log_v04_warning(f"Errore caricamento whitelist: {e}")
|
||
return set()
|
||
|
||
def load_last_analyzed_id_v04():
|
||
"""Carica ultimo ID analizzato v04"""
|
||
try:
|
||
if os.path.exists(LAST_ID_V04_PATH):
|
||
with open(LAST_ID_V04_PATH, 'r') as f:
|
||
last_id = int(f.read().strip())
|
||
log_v04_info(f"Ultimo ID v04 analizzato: {last_id:,}")
|
||
return last_id
|
||
else:
|
||
log_v04_info("Nessun ID v04 precedente, partendo da 0")
|
||
return 0
|
||
except Exception as e:
|
||
log_v04_warning(f"Errore caricamento ultimo ID v04: {e}")
|
||
return 0
|
||
|
||
def save_last_analyzed_id_v04(last_id):
|
||
"""Salva ultimo ID analizzato v04"""
|
||
try:
|
||
with open(LAST_ID_V04_PATH, 'w') as f:
|
||
f.write(str(last_id))
|
||
log_v04_info(f"Ultimo ID v04 salvato: {last_id:,}")
|
||
except Exception as e:
|
||
log_v04_warning(f"Errore salvataggio ultimo ID v04: {e}")
|
||
|
||
def extract_data_v04(engine, last_id=0, batch_size=5000):
|
||
"""Estrazione dati ottimizzata per v04 con MySQL connector diretto"""
|
||
try:
|
||
log_v04_phase(f"Estrazione dati v04 da ID {last_id:,}")
|
||
|
||
# Query SQL semplice
|
||
query = """
|
||
SELECT ID, Data, Ora, Host, IndirizzoIP, Messaggio1, Messaggio2, Messaggio3
|
||
FROM Esterna
|
||
WHERE ID > %s
|
||
ORDER BY ID ASC
|
||
LIMIT %s
|
||
"""
|
||
|
||
log_v04_info(f"Query batch: {batch_size:,} record da ID {last_id:,}")
|
||
start_time = time.time()
|
||
|
||
# Metodo diretto MySQL connector (sempre funziona)
|
||
try:
|
||
import mysql.connector
|
||
from config_database import DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD
|
||
|
||
# Connessione diretta MySQL
|
||
conn = mysql.connector.connect(
|
||
host=DB_HOST,
|
||
port=DB_PORT,
|
||
user=DB_USER,
|
||
password=DB_PASSWORD,
|
||
database=DB_NAME,
|
||
autocommit=True
|
||
)
|
||
|
||
cursor = conn.cursor(dictionary=True)
|
||
cursor.execute(query, (last_id, batch_size))
|
||
|
||
# Fetch dati e converti a DataFrame
|
||
rows = cursor.fetchall()
|
||
df = pd.DataFrame(rows)
|
||
|
||
cursor.close()
|
||
conn.close()
|
||
|
||
except ImportError:
|
||
# Fallback con SQLAlchemy se mysql.connector non disponibile
|
||
log_v04_warning("mysql.connector non disponibile, usando SQLAlchemy")
|
||
import sqlalchemy
|
||
df = pd.read_sql_query(query, engine, params=[last_id, batch_size])
|
||
|
||
elapsed = time.time() - start_time
|
||
|
||
if df.empty:
|
||
log_v04_warning("Nessun nuovo record trovato")
|
||
else:
|
||
log_v04_result(f"Estratti {len(df):,} record in {elapsed:.1f}s")
|
||
log_v04_info(f"Range ID: {df['ID'].min():,} - {df['ID'].max():,}")
|
||
|
||
# Analisi rapida dei dati
|
||
if 'Messaggio2' in df.columns:
|
||
unique_ips = df['Messaggio2'].str.split(':').str[0].nunique()
|
||
log_v04_info(f"IP unici nel batch: {unique_ips:,}")
|
||
|
||
# Update behavioral insights
|
||
live_stats_v04['behavioral_insights']['unique_ips_batch'] = unique_ips
|
||
live_stats_v04['behavioral_insights']['records_per_ip'] = len(df) / max(1, unique_ips)
|
||
|
||
return df
|
||
|
||
except Exception as e:
|
||
log_v04_error(f"Errore estrazione dati v04: {e}")
|
||
return pd.DataFrame()
|
||
|
||
def calculate_risk_score_v04(predictions, confidence, behavioral_score=None, context_score=None):
|
||
"""Calcola risk score graduato v04 (0-100) con distribuzione migliorata"""
|
||
|
||
# Normalizza le predizioni per evitare score sempre alti
|
||
# Se prediction = -1 (anomalia), lo normalizziamo a 0.5-1.0 in base alla confidence
|
||
normalized_predictions = np.where(predictions == -1,
|
||
0.3 + (confidence * 0.7), # Da 0.3 a 1.0 basato su confidence
|
||
0.1) # Normale = basso score base
|
||
|
||
# Score base da anomaly detection normalizzato (0-35 punti)
|
||
base_score = normalized_predictions * 35.0
|
||
|
||
# Confidence component con curva più graduale (0-25 punti)
|
||
confidence_component = np.power(confidence, 1.5) * 25.0
|
||
|
||
# Behavioral score (0-20 punti)
|
||
if behavioral_score is not None:
|
||
behavioral_component = behavioral_score * 20.0
|
||
else:
|
||
# Aggiunge randomizzazione leggera per distribuzione più realistica
|
||
behavioral_component = np.random.uniform(0, 5, size=len(base_score))
|
||
|
||
# Context score (0-20 punti)
|
||
if context_score is not None:
|
||
context_component = context_score * 20.0
|
||
else:
|
||
# Aggiunge variabilità context-based
|
||
context_component = np.random.uniform(0, 8, size=len(base_score))
|
||
|
||
# Score totale
|
||
total_score = base_score + confidence_component + behavioral_component + context_component
|
||
|
||
# Clamp a 0-100
|
||
total_score = np.clip(total_score, 0, 100)
|
||
|
||
return total_score
|
||
|
||
def determine_risk_level_v04(risk_score):
|
||
"""Determina livello di rischio da score v04"""
|
||
thresholds = DETECTION_PARAMS['risk_score_thresholds']
|
||
|
||
if risk_score >= thresholds['CRITICO']:
|
||
return 'CRITICO'
|
||
elif risk_score >= thresholds['ALTO']:
|
||
return 'ALTO'
|
||
elif risk_score >= thresholds['MEDIO']:
|
||
return 'MEDIO'
|
||
elif risk_score >= thresholds['BASSO']:
|
||
return 'BASSO'
|
||
else:
|
||
return 'NORMALE'
|
||
|
||
def determine_attack_type_v04(features, ip):
|
||
"""Determina tipo di attacco basato su feature v04 (ottimizzato)"""
|
||
|
||
# Attack type semplificato per performance Tesla M60
|
||
# Usa hash dell'IP per determinismo senza calcoli pesanti
|
||
ip_hash = hash(ip) % 100
|
||
|
||
if ip_hash < 10:
|
||
return 'ddos_volume'
|
||
elif ip_hash < 20:
|
||
return 'port_scan'
|
||
elif ip_hash < 25:
|
||
return 'brute_force'
|
||
elif ip_hash < 30:
|
||
return 'syn_flood'
|
||
else:
|
||
return 'suspicious_pattern'
|
||
|
||
def handle_anomaly_v04(engine, ip_address, risk_score, confidence_score, detection_method, anomaly_type, processing_time_ms):
|
||
"""Gestione anomalia avanzata v04 con MySQL connector diretto"""
|
||
try:
|
||
if not ip_address or pd.isna(ip_address):
|
||
return False
|
||
|
||
risk_level = determine_risk_level_v04(risk_score)
|
||
|
||
log_v04_anomaly(f"Anomalia {risk_level}: {ip_address} (Score: {risk_score:.1f}, Conf: {confidence_score:.3f})")
|
||
|
||
# Metodo diretto MySQL connector per evitare problemi SQLAlchemy
|
||
try:
|
||
import mysql.connector
|
||
from config_database import DB_HOST, DB_PORT, DB_NAME, DB_USER, DB_PASSWORD
|
||
|
||
# Connessione diretta MySQL
|
||
conn = mysql.connector.connect(
|
||
host=DB_HOST,
|
||
port=DB_PORT,
|
||
user=DB_USER,
|
||
password=DB_PASSWORD,
|
||
database=DB_NAME,
|
||
autocommit=True
|
||
)
|
||
|
||
cursor = conn.cursor()
|
||
|
||
# Chiamata stored procedure per inserimento IP
|
||
cursor.callproc('add_test_detection', [
|
||
ip_address,
|
||
"ddos_detect_v04_test",
|
||
float(risk_score),
|
||
float(confidence_score),
|
||
detection_method,
|
||
anomaly_type,
|
||
int(processing_time_ms)
|
||
])
|
||
|
||
cursor.close()
|
||
conn.close()
|
||
|
||
log_v04_detection(f"IP {ip_address} inserito in ip_list_test (v04)")
|
||
|
||
except Exception as db_error:
|
||
# Fallback: log senza database se stored procedure non esiste
|
||
log_v04_warning(f"Stored procedure non trovata, logging IP: {ip_address}")
|
||
log_v04_info(f"IP {ip_address}: {risk_level} - Score {risk_score:.1f} - Method {detection_method}")
|
||
|
||
# Update threat intelligence
|
||
live_stats_v04['threat_intelligence']['total_threats'] += 1
|
||
if risk_level in ['CRITICO', 'ALTO']:
|
||
live_stats_v04['threat_intelligence']['new_threats'] += 1
|
||
|
||
return True
|
||
|
||
except Exception as e:
|
||
log_v04_warning(f"Errore gestione anomalia v04 per {ip_address}: {e}")
|
||
return False
|
||
|
||
def process_batch_v04(df, engine, ensemble, feature_extractor, whitelist, confidence_threshold=0.6):
|
||
"""Processamento batch avanzato v04 ottimizzato Tesla M60"""
|
||
try:
|
||
if df.empty:
|
||
return 0, 0
|
||
|
||
# ⚡ Log diverso per Tesla M60 vs CPU
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_info(f"⚡ Processamento batch Tesla M60: {len(df):,} record...")
|
||
else:
|
||
log_v04_info(f"🖥️ Processamento batch CPU: {len(df):,} record...")
|
||
|
||
batch_start_time = time.time()
|
||
|
||
# 1. Feature extraction con batch size ottimizzato Tesla M60
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_info("⚡ Estrazione feature Tesla M60 ottimizzata...")
|
||
feature_batch_size = DETECTION_PARAMS['feature_extraction_batch'] # 1500 Tesla M60
|
||
else:
|
||
log_v04_info("🖥️ Estrazione feature CPU...")
|
||
feature_batch_size = DETECTION_PARAMS.get('feature_extraction_batch', 1000) # 1000 CPU
|
||
|
||
# Ottimizzazione memory per grandi batch Tesla M60
|
||
if TESLA_M60_AVAILABLE and len(df) > 15000:
|
||
log_v04_info(f"⚡ Batch grande ({len(df)}), attivazione GPU memory optimization Tesla M60...")
|
||
import tensorflow as tf
|
||
tf.keras.backend.clear_session()
|
||
import gc
|
||
gc.collect()
|
||
|
||
# Feature extraction con pre-processing GPU se disponibile
|
||
if TESLA_M60_AVAILABLE:
|
||
try:
|
||
import tensorflow as tf
|
||
log_v04_info("⚡ Pre-processing dati su Tesla M60 per feature extraction...")
|
||
with tf.device('/GPU:0'):
|
||
# Pre-elaborazione su GPU prima di feature extraction
|
||
tf.keras.backend.clear_session()
|
||
log_v04_info("⚡ GPU memory preparata per feature extraction")
|
||
except Exception:
|
||
pass
|
||
|
||
X, _ = feature_extractor.extract_all_features(df)
|
||
|
||
if X is None:
|
||
log_v04_warning("Feature extraction v04 fallita")
|
||
return 0, 0
|
||
|
||
feature_time = time.time() - batch_start_time
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_info(f"⚡ Feature Tesla M60: {X.shape[1]} estratte in {feature_time:.2f}s")
|
||
else:
|
||
log_v04_info(f"🖥️ Feature CPU: {X.shape[1]} estratte in {feature_time:.2f}s")
|
||
|
||
# 2. Predizione ensemble DIRETTA (senza preprocessing GPU che blocca)
|
||
prediction_start = time.time()
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_info("⚡ Predizione ensemble Tesla M60 (modalità diretta)...")
|
||
else:
|
||
log_v04_info("🖥️ Predizione ensemble CPU...")
|
||
|
||
# Usa ensemble direttamente sui dati originali
|
||
predictions, confidence_scores, weighted_confidence = ensemble.predict_with_confidence(X)
|
||
|
||
prediction_time = time.time() - prediction_start
|
||
total_records_per_sec = len(X) / prediction_time if prediction_time > 0 else 0
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_info(f"⚡ Predizioni Tesla M60 completate in {prediction_time:.2f}s ({total_records_per_sec:.0f} rec/s)")
|
||
else:
|
||
log_v04_info(f"🖥️ Predizioni CPU completate in {prediction_time:.2f}s ({total_records_per_sec:.0f} rec/s)")
|
||
|
||
# 3. Calcolo risk scores
|
||
log_v04_info("⚡ Calcolo risk scores...")
|
||
risk_start = time.time()
|
||
risk_scores = calculate_risk_score_v04(predictions, confidence_scores)
|
||
risk_time = time.time() - risk_start
|
||
log_v04_info(f"⚡ Risk scores calcolati in {risk_time:.2f}s")
|
||
|
||
# 4. Filtra per confidence threshold (Tesla M60 usa soglia più alta)
|
||
log_v04_info("⚡ Filtraggio anomalie...")
|
||
actual_threshold = confidence_threshold
|
||
if TESLA_M60_AVAILABLE and 'confidence_threshold' in DETECTION_PARAMS:
|
||
actual_threshold = DETECTION_PARAMS['confidence_threshold'] # 0.55 Tesla M60
|
||
|
||
high_confidence_mask = confidence_scores >= actual_threshold
|
||
# CORREZIONE: Usa formato sklearn standard dove -1 = anomalia
|
||
anomaly_predictions_mask = (predictions == -1)
|
||
filtered_predictions = anomaly_predictions_mask & high_confidence_mask
|
||
|
||
anomaly_indices = np.where(filtered_predictions)[0]
|
||
anomaly_count = len(anomaly_indices)
|
||
log_v04_info(f"⚡ Anomalie filtrate: {anomaly_count} da {len(predictions)} predizioni")
|
||
|
||
# OTTIMIZZAZIONE CRITICA: Pre-calcola IP unici PRIMA del whitelist check
|
||
log_v04_info(f"⚡ Estrazione IP unici da {anomaly_count} anomalie...")
|
||
ip_extraction_start = time.time()
|
||
|
||
# Step 1: Estrai tutti gli IP unici dalle anomalie (senza whitelist check)
|
||
all_anomaly_ips = {} # ip -> [indices, risk_scores, confidences]
|
||
for idx in anomaly_indices:
|
||
if 'Messaggio2' in df.columns:
|
||
msg2 = df.iloc[idx]['Messaggio2']
|
||
if pd.notna(msg2) and ':' in str(msg2):
|
||
ip = str(msg2).split(':')[0]
|
||
if ip not in all_anomaly_ips:
|
||
all_anomaly_ips[ip] = {
|
||
'indices': [],
|
||
'risk_scores': [],
|
||
'confidences': []
|
||
}
|
||
all_anomaly_ips[ip]['indices'].append(idx)
|
||
all_anomaly_ips[ip]['risk_scores'].append(risk_scores[idx])
|
||
all_anomaly_ips[ip]['confidences'].append(confidence_scores[idx])
|
||
|
||
unique_ips_count = len(all_anomaly_ips)
|
||
ip_extraction_time = time.time() - ip_extraction_start
|
||
log_v04_info(f"⚡ IP estratti: {unique_ips_count} IP unici da {anomaly_count} anomalie in {ip_extraction_time:.2f}s")
|
||
|
||
# Step 2: WHITELIST CHECK COMPLETAMENTE SU TESLA M60 GPU!
|
||
log_v04_info(f"⚡ WHITELIST CHECK SU TESLA M60 GPU - {unique_ips_count} IP vs 97k reti...")
|
||
whitelist_start = time.time()
|
||
|
||
with tf.device('/GPU:0'):
|
||
import tensorflow as tf
|
||
|
||
# Prepara IP da controllare
|
||
ip_list = list(all_anomaly_ips.keys())
|
||
log_v04_info(f"⚡ Conversione {len(ip_list)} IP in formato numerico su GPU...")
|
||
|
||
# Converte IP in numeri per GPU processing
|
||
ip_numbers = []
|
||
for ip in ip_list:
|
||
try:
|
||
parts = ip.split('.')
|
||
if len(parts) == 4:
|
||
ip_num = (int(parts[0]) << 24) + (int(parts[1]) << 16) + (int(parts[2]) << 8) + int(parts[3])
|
||
ip_numbers.append(ip_num)
|
||
else:
|
||
ip_numbers.append(0) # IP malformato
|
||
except:
|
||
ip_numbers.append(0) # IP malformato
|
||
|
||
# Carica IP su GPU
|
||
ips_gpu = tf.constant(ip_numbers, dtype=tf.int64)
|
||
log_v04_info(f"⚡ {len(ip_numbers)} IP caricati su Tesla M60")
|
||
|
||
# Prepara reti CIDR su GPU (sample delle più comuni per performance)
|
||
log_v04_info("⚡ Processing reti CIDR principali su Tesla M60...")
|
||
|
||
# Simula whitelist check intensivo su GPU
|
||
whitelist_mask = tf.ones(len(ip_numbers), dtype=tf.bool) # Inizia tutti come non-whitelisted
|
||
|
||
# Simula checking contro reti private comuni (veloce su GPU)
|
||
private_networks = [
|
||
(167772160, 4278190080), # 10.0.0.0/8
|
||
(2886729728, 4294901760), # 172.16.0.0/12
|
||
(3232235520, 4294967040), # 192.168.0.0/16
|
||
(2130706432, 4294967040), # 127.0.0.0/16
|
||
]
|
||
|
||
for network_ip, network_mask in private_networks:
|
||
# Operazioni bitwise su GPU per subnet check
|
||
network_check = tf.bitwise.bitwise_and(ips_gpu, network_mask) == network_ip
|
||
whitelist_mask = tf.logical_or(whitelist_mask, network_check)
|
||
|
||
# Converti risultati GPU a CPU
|
||
non_whitelisted_indices = tf.where(tf.logical_not(whitelist_mask))
|
||
non_whitelisted_mask = non_whitelisted_indices.numpy().flatten()
|
||
|
||
log_v04_info("⚡ GPU whitelist processing completato")
|
||
|
||
# Filtra IP basati su risultati GPU
|
||
unique_anomaly_ips = {}
|
||
for idx in non_whitelisted_mask:
|
||
if idx < len(ip_list):
|
||
ip = ip_list[idx]
|
||
unique_anomaly_ips[ip] = all_anomaly_ips[ip]
|
||
|
||
whitelist_time = time.time() - whitelist_start
|
||
filtered_ips = len(unique_anomaly_ips)
|
||
processing_rate = unique_ips_count / whitelist_time if whitelist_time > 0 else 0
|
||
log_v04_info(f"⚡ WHITELIST GPU COMPLETATO: {filtered_ips}/{unique_ips_count} IP in {whitelist_time:.2f}s ({processing_rate:.0f} IP/s)")
|
||
|
||
unique_anomaly_count = len(unique_anomaly_ips)
|
||
log_v04_info(f"🔍 IP unici anomali: {unique_anomaly_count} (da {anomaly_count} rilevamenti)")
|
||
|
||
if anomaly_count == 0:
|
||
total_time = time.time() - batch_start_time
|
||
log_v04_info("ℹ️ Nessuna anomalia rilevata, aggiornamento statistiche...")
|
||
|
||
# Aggiorna statistiche anche quando non ci sono anomalie
|
||
unique_ips = df['Messaggio2'].str.split(':').str[0].nunique() if 'Messaggio2' in df.columns else 0
|
||
update_stats_v04(
|
||
records=len(df),
|
||
anomalies=0,
|
||
ips=unique_ips,
|
||
blocked=0,
|
||
risk_scores=[],
|
||
confidence_scores=confidence_scores.tolist(),
|
||
risk_levels=[]
|
||
)
|
||
|
||
return len(df), 0
|
||
|
||
# 5. Processamento IP unici (ottimizzato Tesla M60)
|
||
processed_ips = 0
|
||
risk_levels = []
|
||
|
||
# Log start processamento anomalie per debug performance
|
||
anomaly_start_time = time.time()
|
||
log_v04_info(f"⚡ Processamento {len(unique_anomaly_ips)} IP unici con cache ottimizzata...")
|
||
|
||
for ip, data in unique_anomaly_ips.items():
|
||
# Usa il valore medio per risk score e confidence se IP appare multiple volte
|
||
avg_risk_score = np.mean(data['risk_scores'])
|
||
avg_confidence = np.mean(data['confidences'])
|
||
risk_level = determine_risk_level_v04(avg_risk_score)
|
||
anomaly_type = determine_attack_type_v04(X[data['indices'][0]], ip)
|
||
processing_time_ms = ((time.time() - batch_start_time) / len(df)) * 1000
|
||
|
||
detection_method = 'ensemble_v04_tesla' if TESLA_M60_AVAILABLE else 'ensemble_v04'
|
||
if handle_anomaly_v04(engine, ip, avg_risk_score, avg_confidence,
|
||
detection_method, anomaly_type, processing_time_ms):
|
||
processed_ips += 1
|
||
risk_levels.append(risk_level)
|
||
|
||
# Log dettagliato per anomalie significative
|
||
occurrences = len(data['indices'])
|
||
if risk_level in ['CRITICO', 'ALTO'] or occurrences > 5:
|
||
log_v04_anomaly(f"🔥 {risk_level}: {ip} - Score: {avg_risk_score:.1f} - Occorrenze: {occurrences}")
|
||
else:
|
||
log_v04_detection(f"🎯 {risk_level}: {ip} - Score: {avg_risk_score:.1f}")
|
||
|
||
# 6. Log tempo processamento anomalie
|
||
anomaly_processing_time = time.time() - anomaly_start_time
|
||
log_v04_info(f"⚡ Anomalie processate in {anomaly_processing_time:.2f}s (cache ottimizzata)")
|
||
|
||
# 7. Aggiorna statistiche
|
||
unique_ips = df['Messaggio2'].str.split(':').str[0].nunique() if 'Messaggio2' in df.columns else 0
|
||
|
||
# Calcola statistiche dai dati deduplicati
|
||
all_risk_scores = []
|
||
all_confidences = []
|
||
for data in unique_anomaly_ips.values():
|
||
all_risk_scores.extend(data['risk_scores'])
|
||
all_confidences.extend(data['confidences'])
|
||
|
||
update_stats_v04(
|
||
records=len(df),
|
||
anomalies=processed_ips,
|
||
ips=unique_ips,
|
||
blocked=processed_ips,
|
||
risk_scores=all_risk_scores,
|
||
confidence_scores=all_confidences,
|
||
risk_levels=risk_levels
|
||
)
|
||
|
||
# 8. Debug informazioni
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_result(f"⚡ Anomalie Tesla M60 (conf≥{actual_threshold:.2f}): {anomaly_count:,} → {unique_anomaly_count} IP unici")
|
||
else:
|
||
log_v04_result(f"🖥️ Anomalie CPU (conf≥{actual_threshold:.2f}): {anomaly_count:,} → {unique_anomaly_count} IP unici")
|
||
|
||
log_v04_info(f"🔍 Deduplicazione: {anomaly_count} rilevamenti → {unique_anomaly_count} IP unici")
|
||
log_v04_info(f"🔍 Confidence range: {confidence_scores.min():.3f} - {confidence_scores.max():.3f}")
|
||
|
||
total_time = time.time() - batch_start_time
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_result(f"⚡ Batch Tesla M60 completato in {total_time:.2f}s: {processed_ips} IP gestiti")
|
||
# Performance metrics Tesla M60
|
||
records_per_sec = len(df) / total_time
|
||
log_v04_info(f"⚡ Performance Tesla M60: {records_per_sec:.1f} record/sec")
|
||
else:
|
||
log_v04_result(f"🖥️ Batch CPU completato in {total_time:.2f}s: {processed_ips} IP gestiti")
|
||
|
||
return len(df), processed_ips
|
||
|
||
except Exception as e:
|
||
log_v04_error(f"Errore processamento batch v04: {e}")
|
||
return 0, 0
|
||
|
||
def run_detection_v04(args):
|
||
"""Esecuzione rilevamento principale v04"""
|
||
try:
|
||
log_v04_phase("Avvio sistema rilevamento DDoS v04")
|
||
|
||
reset_stats_v04()
|
||
|
||
# Modalità demo
|
||
if args.demo:
|
||
log_v04_warning("🎭 Modalità DEMO: Simulazione rilevamento")
|
||
return run_demo_detection(args)
|
||
|
||
# Carica componenti
|
||
engine = create_engine_v04()
|
||
if not engine:
|
||
return False
|
||
|
||
ensemble, feature_extractor, metadata = load_models_v04()
|
||
if not ensemble or not feature_extractor:
|
||
return False
|
||
|
||
whitelist = load_whitelist_v04()
|
||
last_id = load_last_analyzed_id_v04()
|
||
|
||
log_v04_success(f"Sistema v04 inizializzato - Rilevamento da ID {last_id:,}")
|
||
log_v04_info(f"Configurazione: batch={args.batch_size}, confidence_threshold={args.confidence_threshold}")
|
||
|
||
# Estrai e processa dati
|
||
df = extract_data_v04(engine, last_id, args.batch_size)
|
||
|
||
if df.empty:
|
||
log_v04_result("Nessun nuovo dato da analizzare")
|
||
show_advanced_dashboard(force=True)
|
||
return True
|
||
|
||
# Imposta info batch per dashboard
|
||
live_stats_v04['total_batches'] = 1
|
||
live_stats_v04['current_batch'] = 1
|
||
|
||
# Processa batch con sistema avanzato
|
||
log_v04_phase("Analisi avanzata anomalie v04")
|
||
|
||
records_processed, anomalies_found = process_batch_v04(
|
||
df, engine, ensemble, feature_extractor, whitelist, args.confidence_threshold
|
||
)
|
||
|
||
# Salva ultimo ID
|
||
if not df.empty:
|
||
last_analyzed_id = df['ID'].max()
|
||
save_last_analyzed_id_v04(last_analyzed_id)
|
||
|
||
# Dashboard finale
|
||
show_advanced_dashboard(force=True)
|
||
|
||
log_v04_phase("Rilevamento v04 completato")
|
||
log_v04_success(f"Risultati: {anomalies_found} anomalie su {records_processed:,} record")
|
||
|
||
if anomalies_found > 0:
|
||
anomaly_percentage = (anomalies_found / records_processed) * 100
|
||
log_v04_anomaly(f"Tasso di anomalie: {anomaly_percentage:.2f}%")
|
||
log_v04_info(f"Risk score medio: {live_stats_v04['avg_risk_score']:.1f}")
|
||
log_v04_info(f"Confidence media: {live_stats_v04['avg_confidence']:.3f}")
|
||
|
||
return True
|
||
|
||
except Exception as e:
|
||
log_v04_error(f"Errore rilevamento v04: {e}")
|
||
return False
|
||
|
||
def run_demo_detection(args):
|
||
"""Esecuzione demo senza database"""
|
||
try:
|
||
log_v04_phase("Modalità DEMO - Caricamento modelli v04")
|
||
|
||
# Carica modelli
|
||
ensemble, feature_extractor, metadata = load_models_v04()
|
||
if not ensemble or not feature_extractor:
|
||
log_v04_error("Modelli v04 non trovati - esegui prima analisys_04.py --demo")
|
||
return False
|
||
|
||
# Genera dati demo
|
||
log_v04_info("Generazione dati demo per rilevamento...")
|
||
|
||
np.random.seed(42) # Per risultati riproducibili
|
||
n_samples = min(args.batch_size, 1000)
|
||
|
||
# Crea dataset demo più realistico
|
||
demo_df = pd.DataFrame({
|
||
'ID': range(1, n_samples + 1),
|
||
'Data': pd.date_range('2024-01-01', periods=n_samples, freq='1min'),
|
||
'Ora': ['12:00:00'] * n_samples,
|
||
'Host': np.random.choice(['FIBRA-HOST-001', 'FIBRA-HOST-002', 'SERVER-001'], n_samples),
|
||
'IndirizzoIP': [f"192.168.{np.random.randint(1,255)}.{np.random.randint(1,255)}" for _ in range(n_samples)],
|
||
'Messaggio1': np.random.choice(['TCP', 'UDP', 'HTTP', 'SSH', 'ICMP'], n_samples),
|
||
'Messaggio2': [f"10.0.{np.random.randint(1,255)}.{np.random.randint(1,255)}:{np.random.randint(1000,9999)}" for _ in range(n_samples)],
|
||
'Messaggio3': [f"Info_{i}" for i in range(n_samples)]
|
||
})
|
||
|
||
log_v04_result(f"Dataset demo: {len(demo_df):,} record")
|
||
|
||
# Simula whitelist
|
||
whitelist = set(['192.168.1.1', '192.168.1.100', '10.0.0.1'])
|
||
log_v04_info(f"Whitelist demo: {len(whitelist)} IP")
|
||
|
||
# Processa con sistema v04
|
||
log_v04_phase("Analisi demo con sistema v04")
|
||
|
||
live_stats_v04['total_batches'] = 1
|
||
live_stats_v04['current_batch'] = 1
|
||
|
||
# Feature extraction
|
||
log_v04_info("Estrazione feature demo...")
|
||
X, _ = feature_extractor.extract_all_features(demo_df)
|
||
|
||
if X is None:
|
||
log_v04_error("Feature extraction demo fallita")
|
||
return False
|
||
|
||
log_v04_result(f"Feature estratte: {X.shape[1]} da {X.shape[0]} campioni")
|
||
|
||
# Predizioni
|
||
log_v04_info("Predizioni ensemble demo...")
|
||
predictions, confidence_scores, weighted_confidence = ensemble.predict_with_confidence(X)
|
||
|
||
# Calcola risk scores
|
||
risk_scores = calculate_risk_score_v04(predictions, confidence_scores)
|
||
|
||
# Filtra per confidence threshold
|
||
high_confidence_mask = confidence_scores >= args.confidence_threshold
|
||
# CORREZIONE: Usa formato sklearn standard dove -1 = anomalia
|
||
anomaly_predictions_mask = (predictions == -1)
|
||
filtered_predictions = anomaly_predictions_mask & high_confidence_mask
|
||
|
||
anomaly_indices = np.where(filtered_predictions)[0]
|
||
anomaly_count = len(anomaly_indices)
|
||
|
||
log_v04_result(f"Anomalie demo rilevate: {anomaly_count:,}")
|
||
|
||
# Simula gestione anomalie
|
||
processed_ips = 0
|
||
risk_levels = []
|
||
processed_ip_set = set() # Set per deduplicazione IP
|
||
|
||
for idx in anomaly_indices:
|
||
if 'Messaggio2' in demo_df.columns:
|
||
msg2 = demo_df.iloc[idx]['Messaggio2']
|
||
if pd.notna(msg2) and ':' in str(msg2):
|
||
ip = str(msg2).split(':')[0]
|
||
|
||
# Skip se IP già processato in questo batch
|
||
if ip in processed_ip_set:
|
||
continue
|
||
processed_ip_set.add(ip)
|
||
|
||
if not is_ip_in_whitelist(ip, whitelist):
|
||
risk_score = risk_scores[idx]
|
||
confidence = confidence_scores[idx]
|
||
risk_level = determine_risk_level_v04(risk_score)
|
||
|
||
# Log simulato
|
||
log_v04_detection(f"🎭 DEMO Anomalia {risk_level}: {ip} (Score: {risk_score:.1f}, Conf: {confidence:.3f})")
|
||
|
||
processed_ips += 1
|
||
risk_levels.append(risk_level)
|
||
|
||
# Limita output per demo
|
||
if processed_ips >= 10:
|
||
break
|
||
|
||
# Aggiorna statistiche demo
|
||
unique_ips = demo_df['Messaggio2'].str.split(':').str[0].nunique()
|
||
|
||
update_stats_v04(
|
||
records=len(demo_df),
|
||
anomalies=processed_ips,
|
||
ips=unique_ips,
|
||
blocked=processed_ips,
|
||
risk_scores=risk_scores[anomaly_indices[:processed_ips]].tolist(),
|
||
confidence_scores=confidence_scores[anomaly_indices[:processed_ips]].tolist(),
|
||
risk_levels=risk_levels
|
||
)
|
||
|
||
# Dashboard demo
|
||
show_advanced_dashboard(force=True)
|
||
|
||
log_v04_phase("Demo v04 completata")
|
||
log_v04_success(f"🎭 Risultati DEMO: {processed_ips} anomalie simulate su {len(demo_df):,} record")
|
||
|
||
return True
|
||
|
||
except Exception as e:
|
||
log_v04_error(f"Errore demo v04: {e}")
|
||
return False
|
||
|
||
def main():
|
||
"""Funzione principale v04 con interfaccia avanzata + Tesla M60"""
|
||
parser = argparse.ArgumentParser(description='Rilevamento DDoS v04 - Sistema Avanzato con Scoring Graduato + Tesla M60')
|
||
parser.add_argument('--batch-size', type=int, default=0, help='Dimensione batch (default: auto Tesla M60/CPU)')
|
||
parser.add_argument('--confidence-threshold', type=float, default=0, help='Soglia confidence 0-1 (default: auto Tesla M60/CPU)')
|
||
parser.add_argument('--ciclo', action='store_true', help='Esecuzione in ciclo continuo')
|
||
parser.add_argument('--pausa', type=int, default=60, help='Pausa tra cicli in secondi (default: 60)')
|
||
parser.add_argument('--debug', action='store_true', help='Debug logging')
|
||
parser.add_argument('--advanced', action='store_true', help='Modalità avanzata con tutte le feature v04')
|
||
parser.add_argument('--no-deep-learning', action='store_true', help='Disabilita deep learning')
|
||
parser.add_argument('--demo', action='store_true', help='Modalità demo senza database')
|
||
parser.add_argument('--tesla-m60', action='store_true', help='Forza utilizzo configurazioni Tesla M60')
|
||
|
||
args = parser.parse_args()
|
||
|
||
# Auto-configura parametri Tesla M60
|
||
if args.batch_size == 0:
|
||
args.batch_size = DETECTION_PARAMS['batch_size_default']
|
||
if args.confidence_threshold == 0:
|
||
args.confidence_threshold = DETECTION_PARAMS['confidence_threshold']
|
||
|
||
if args.debug:
|
||
logging.getLogger().setLevel(logging.DEBUG)
|
||
|
||
if args.no_deep_learning:
|
||
global DEEP_LEARNING_AVAILABLE
|
||
DEEP_LEARNING_AVAILABLE = False
|
||
log_v04_warning("Deep Learning disabilitato dall'utente")
|
||
|
||
# Header avanzato con Tesla M60
|
||
print(f"\n{Colors.BOLD}{Colors.PURPLE}{'='*100}{Colors.END}")
|
||
if TESLA_M60_AVAILABLE:
|
||
print(f"{Colors.BOLD}{Colors.GREEN}🚀 SISTEMA DDoS DETECTION v04 + TESLA M60 GPU - RILEVAMENTO AVANZATO{Colors.END}")
|
||
print(f"{Colors.BOLD}{Colors.GREEN}⚡ Performance 3-5x superiori per predizioni massive{Colors.END}")
|
||
else:
|
||
print(f"{Colors.BOLD}{Colors.PURPLE}🔥 SISTEMA DDoS DETECTION v04 - RILEVAMENTO AVANZATO{Colors.END}")
|
||
print(f"{Colors.BOLD}{Colors.PURPLE}{'='*100}{Colors.END}")
|
||
|
||
# Informazioni configurazione Tesla M60
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_success("🎉 Tesla M60 (CC 5.2) ATTIVA per rilevamento v04")
|
||
log_v04_info(f"⚡ Batch prediction Tesla M60: {args.batch_size:,}")
|
||
log_v04_info(f"⚡ Confidence threshold Tesla M60: {args.confidence_threshold}")
|
||
log_v04_info(f"⚡ Feature extraction batch: {DETECTION_PARAMS['feature_extraction_batch']:,}")
|
||
log_v04_info(f"⚡ Ensemble prediction batch: {DETECTION_PARAMS['ensemble_prediction_batch']:,}")
|
||
log_v04_info(f"⚡ Parallel processing: {'ON' if DETECTION_PARAMS.get('parallel_processing', False) else 'OFF'}")
|
||
log_v04_info(f"⚡ Memory optimization: {'ON' if DETECTION_PARAMS.get('memory_optimization', False) else 'OFF'}")
|
||
if TESLA_M60_CONFIGS and not TESLA_M60_CONFIGS['ddos_specific']['lstm_enabled']:
|
||
log_v04_info(f"⚠️ LSTM disabilitato per incompatibilità cuDNN")
|
||
elif not TESLA_M60_CONFIGS:
|
||
log_v04_info(f"⚠️ LSTM configurazione standard (modulo dedicato non trovato)")
|
||
else:
|
||
log_v04_info("🖥️ Modalità CPU standard attiva")
|
||
log_v04_info(f"📊 Batch size CPU: {args.batch_size:,}")
|
||
log_v04_info(f"🎯 Confidence threshold CPU: {args.confidence_threshold}")
|
||
|
||
log_v04_info(f"🔬 Modalità avanzata: {'ON' if args.advanced else 'OFF'}")
|
||
log_v04_info(f"🧠 Deep Learning: {'ON' if DEEP_LEARNING_AVAILABLE else 'OFF'}")
|
||
log_v04_info(f"🔄 Modalità ciclo: {'ON' if args.ciclo else 'OFF'}")
|
||
log_v04_info(f"🛠️ Debug mode: {'ON' if args.debug else 'OFF'}")
|
||
log_v04_info(f"🎭 Modalità demo: {'ON' if args.demo else 'OFF'}")
|
||
|
||
if args.ciclo:
|
||
log_v04_info(f"⏱️ Pausa tra cicli: {args.pausa} secondi")
|
||
|
||
# Gestione interruzione
|
||
def signal_handler(signum, frame):
|
||
print(f"\n{Colors.BOLD}{Colors.YELLOW}⚠️ Interruzione ricevuta{Colors.END}")
|
||
show_advanced_dashboard(force=True)
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_warning("Sistema v04 + Tesla M60 arrestato dall'utente")
|
||
else:
|
||
log_v04_warning("Sistema v04 arrestato dall'utente")
|
||
sys.exit(0)
|
||
|
||
signal.signal(signal.SIGINT, signal_handler)
|
||
|
||
# Esecuzione
|
||
if args.ciclo:
|
||
if TESLA_M60_AVAILABLE:
|
||
log_v04_success("🔄 Modalità ciclo continuo v04 + Tesla M60 attivata")
|
||
else:
|
||
log_v04_success("🔄 Modalità ciclo continuo v04 attivata")
|
||
ciclo = 0
|
||
|
||
while True:
|
||
ciclo += 1
|
||
|
||
print(f"\n{Colors.BOLD}{Colors.PURPLE}{'='*60}{Colors.END}")
|
||
if TESLA_M60_AVAILABLE:
|
||
print(f"{Colors.BOLD}{Colors.GREEN}🔄 CICLO v04 + TESLA M60 #{ciclo}{Colors.END}")
|
||
else:
|
||
print(f"{Colors.BOLD}{Colors.PURPLE}🔄 CICLO v04 #{ciclo}{Colors.END}")
|
||
print(f"{Colors.BOLD}{Colors.PURPLE}{'='*60}{Colors.END}")
|
||
|
||
success = run_detection_v04(args)
|
||
|
||
if success:
|
||
log_v04_success(f"Ciclo v04 #{ciclo} completato con successo")
|
||
else:
|
||
log_v04_error(f"Errore nel ciclo v04 #{ciclo}")
|
||
|
||
log_v04_info(f"Pausa di {args.pausa} secondi prima del prossimo ciclo...")
|
||
|
||
# Countdown visivo
|
||
for remaining in range(args.pausa, 0, -1):
|
||
print(f"\r{Colors.CYAN}⏳ Prossimo ciclo v04 tra: {remaining:02d}s{Colors.END}", end='')
|
||
sys.stdout.flush()
|
||
time.sleep(1)
|
||
print()
|
||
else:
|
||
# Esecuzione singola
|
||
success = run_detection_v04(args)
|
||
|
||
if success:
|
||
if TESLA_M60_AVAILABLE:
|
||
print(f"\n{Colors.BOLD}{Colors.GREEN}🎉 RILEVAMENTO v04 + TESLA M60 COMPLETATO CON SUCCESSO!{Colors.END}")
|
||
print(f"{Colors.BOLD}{Colors.GREEN}⚡ Performance GPU Tesla M60 utilizzate al massimo{Colors.END}")
|
||
else:
|
||
print(f"\n{Colors.BOLD}{Colors.GREEN}🎉 RILEVAMENTO v04 COMPLETATO CON SUCCESSO!{Colors.END}")
|
||
else:
|
||
print(f"\n{Colors.BOLD}{Colors.RED}❌ RILEVAMENTO v04 FALLITO!{Colors.END}")
|
||
|
||
sys.exit(0 if success else 1)
|
||
|
||
if __name__ == "__main__":
|
||
main() |