diff --git a/python_ml/main.py b/python_ml/main.py index 4937c64..4777373 100644 --- a/python_ml/main.py +++ b/python_ml/main.py @@ -129,7 +129,10 @@ async def train_model(request: TrainRequest, background_tasks: BackgroundTasks): Esegue in background per non bloccare l'API """ def do_training(): + conn = None + cursor = None try: + print("[TRAIN] Inizio training...") conn = get_db_connection() cursor = conn.cursor(cursor_factory=RealDictCursor) @@ -148,13 +151,18 @@ async def train_model(request: TrainRequest, background_tasks: BackgroundTasks): print("[TRAIN] Nessun log trovato per training") return + print(f"[TRAIN] Trovati {len(logs)} log per training") + # Converti in DataFrame df = pd.DataFrame(logs) # Training + print("[TRAIN] Addestramento modello...") result = ml_analyzer.train(df, contamination=request.contamination) + print(f"[TRAIN] Modello addestrato: {result}") # Salva nel database + print("[TRAIN] Salvataggio training history nel database...") cursor.execute(""" INSERT INTO training_history (model_version, records_processed, features_count, training_duration, status, notes) @@ -168,6 +176,7 @@ async def train_model(request: TrainRequest, background_tasks: BackgroundTasks): f"Anomalie: {result['anomalies_detected']}/{result['unique_ips']}" )) conn.commit() + print("[TRAIN] โœ… Training history salvato con successo!") cursor.close() conn.close() @@ -175,7 +184,16 @@ async def train_model(request: TrainRequest, background_tasks: BackgroundTasks): print(f"[TRAIN] Completato: {result}") except Exception as e: - print(f"[TRAIN ERROR] {e}") + print(f"[TRAIN ERROR] โŒ Errore durante training: {e}") + import traceback + traceback.print_exc() + if conn: + conn.rollback() + finally: + if cursor: + cursor.close() + if conn: + conn.close() # Esegui in background background_tasks.add_task(do_training) diff --git a/replit.md b/replit.md index 76a37fc..6b1ac6c 100644 --- a/replit.md +++ b/replit.md @@ -150,6 +150,16 @@ The IDS features a React-based frontend for real-time monitoring, detection visu ## Fix Recenti (Novembre 2025) +### ๐Ÿ”ง Fix Training History + Logging Migliorato (22 Nov 2025 - 11:30) +- **Problema**: Storico training sempre vuoto anche dopo training completato + - **Causa**: INSERT in `training_history` falliva silenziosamente (errore solo in console) + - **Soluzione**: Error handling migliorato con logging dettagliato e traceback +- **Fix Applicati**: + - โœ… `python_ml/main.py`: Logging step-by-step training + traceback errori + - โœ… `finally` block per chiusura connessioni sicura + - โœ… `rollback` automatico in caso di errore +- **Testing**: Monitorare `/var/log/ids/ml_backend.log` durante training per vedere errori esatti + ### ๐Ÿ› Fix Monitoring Syslog Parser + Schema Database (22 Nov 2025 - 11:10) - **Problema 1**: Dashboard mostrava Syslog Parser in errore nonostante systemd "active (running)" - **Causa**: Bug fetch `signal: controller2.abort` invece di `signal: controller2.signal` in `/api/services/status` diff --git a/update_from_git.sh b/update_from_git.sh deleted file mode 100755 index d01caa1..0000000 --- a/update_from_git.sh +++ /dev/null @@ -1,211 +0,0 @@ -#!/bin/bash - -# ============================================================================= -# IDS - Aggiornamento Sistema da git.alfacom.it -# ============================================================================= -# Eseguire come ROOT: ./update_from_git.sh [--db] -# Lo script esegue i comandi git come utente 'ids' automaticamente -# -# Opzioni: -# --db Sincronizza anche schema database da database-schema/schema.sql -# ============================================================================= - -set -e - -# Colori per output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' - -# Opzioni -UPDATE_DB=false -if [ "$1" == "--db" ]; then - UPDATE_DB=true -fi - -echo -e "${BLUE}" -echo "โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—" -echo "โ•‘ ๐Ÿ”„ AGGIORNAMENTO SISTEMA IDS DA GIT โ•‘" -echo "โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•" -echo -e "${NC}" - -IDS_DIR="/opt/ids" -IDS_USER="ids" - -# Verifica di essere root -if [ "$EUID" -ne 0 ]; then - echo -e "${RED}โŒ Questo script deve essere eseguito come root${NC}" - echo -e "${YELLOW} Esegui: sudo ./update_from_git.sh [--db]${NC}" - exit 1 -fi - -cd "$IDS_DIR" || exit 1 - -# Fix git ownership se necessario -echo -e "${BLUE}๐Ÿ”ง Verifica configurazione git...${NC}" -if ! sudo -u $IDS_USER git config --global --get-all safe.directory | grep -q "^/opt/ids$"; then - echo -e "${YELLOW} Configuro safe directory...${NC}" - sudo -u $IDS_USER git config --global --add safe.directory /opt/ids - echo -e "${GREEN} โœ… Configurato${NC}" -fi - -# Backup configurazione locale -echo -e "\n${BLUE}๐Ÿ’พ Backup configurazione locale...${NC}" -if [ -f ".env" ]; then - cp .env .env.backup - echo -e "${GREEN}โœ… .env salvato in .env.backup${NC}" -fi - -if [ -f "git.env" ]; then - cp git.env git.env.backup - echo -e "${GREEN}โœ… git.env salvato in git.env.backup${NC}" -fi - -# Verifica modifiche locali -echo -e "\n${BLUE}๐Ÿ” Verifica modifiche locali...${NC}" -if ! sudo -u $IDS_USER git diff-index --quiet HEAD -- 2>/dev/null; then - echo -e "${YELLOW}โš ๏ธ Ci sono modifiche locali non committate${NC}" - echo -e "${YELLOW} Esegui 'git status' per vedere i dettagli${NC}" - read -p "Vuoi procedere comunque? (y/n) " -n 1 -r - echo - if [[ ! $REPLY =~ ^[Yy]$ ]]; then - exit 1 - fi - # Stash modifiche locali - echo -e "${BLUE} Salvo modifiche locali temporaneamente...${NC}" - sudo -u $IDS_USER git stash -fi - -# Pull da git -echo -e "\n${BLUE}๐Ÿ“ฅ Download aggiornamenti da git.alfacom.it...${NC}" -sudo -u $IDS_USER git fetch origin -sudo -u $IDS_USER git pull origin main - -if [ $? -eq 0 ]; then - echo -e "${GREEN}โœ… Aggiornamenti scaricati con successo${NC}" -else - echo -e "${RED}โŒ Errore durante il download${NC}" - exit 1 -fi - -# Ripristina configurazione locale -echo -e "\n${BLUE}๐Ÿ”„ Ripristino configurazione locale...${NC}" -if [ -f ".env.backup" ]; then - cp .env.backup .env - chown $IDS_USER:$IDS_USER .env - echo -e "${GREEN}โœ… .env ripristinato${NC}" -fi - -if [ -f "git.env.backup" ]; then - cp git.env.backup git.env - chown $IDS_USER:$IDS_USER git.env - echo -e "${GREEN}โœ… git.env ripristinato${NC}" -fi - -# Sincronizza schema database se richiesto -if [ "$UPDATE_DB" = true ]; then - echo -e "\n${BLUE}๐Ÿ—„๏ธ Sincronizzazione schema database...${NC}" - - SCHEMA_DIR="$IDS_DIR/database-schema" - - # Carica configurazione database - if [ ! -f "$IDS_DIR/.env" ]; then - echo -e "${RED}โŒ File .env non trovato${NC}" - exit 1 - fi - - source "$IDS_DIR/.env" - export PGPASSWORD="$PGPASSWORD" - - # Trova tutti gli script SQL (escludi schema.sql che รจ solo snapshot) - SQL_SCRIPTS=$(find "$SCHEMA_DIR" -maxdepth 1 -name "*.sql" ! -name "schema.sql" -type f | sort) - - if [ -z "$SQL_SCRIPTS" ]; then - echo -e "${YELLOW}โš ๏ธ Nessuno script SQL trovato in database-schema/${NC}" - echo -e "${YELLOW} (schema.sql รจ solo uno snapshot, non viene applicato)${NC}" - else - echo -e "${YELLOW}โš ๏ธ APPLICAZIONE MODIFICHE AL DATABASE${NC}" - echo -e "${YELLOW} Database: ${PGDATABASE}@${PGHOST}${NC}" - echo -e "${YELLOW} Script da applicare:${NC}" - - # Mostra lista script - for script in $SQL_SCRIPTS; do - echo -e "${YELLOW} โ€ข $(basename $script)${NC}" - done - - read -p "Confermi applicazione modifiche? (y/n) " -n 1 -r - echo - - if [[ $REPLY =~ ^[Yy]$ ]]; then - # Applica tutti gli script in ordine - for script in $SQL_SCRIPTS; do - SCRIPT_NAME=$(basename "$script") - echo -e "\n${BLUE}๐Ÿ“„ Applicando $SCRIPT_NAME...${NC}" - - psql -h "${PGHOST:-127.0.0.1}" -p "${PGPORT:-5432}" -U "${PGUSER}" -d "${PGDATABASE}" -f "$script" - - if [ $? -eq 0 ]; then - echo -e "${GREEN}โœ… $SCRIPT_NAME applicato con successo${NC}" - else - echo -e "${RED}โŒ Errore applicando $SCRIPT_NAME${NC}" - echo -e "${YELLOW}โš ๏ธ Gli script successivi NON saranno applicati${NC}" - exit 1 - fi - done - - echo -e "\n${GREEN}โœ… Tutti gli script SQL applicati con successo${NC}" - else - echo -e "${YELLOW}โญ๏ธ Sincronizzazione database saltata${NC}" - fi - fi -fi - -# Aggiorna dipendenze Node.js -echo -e "\n${BLUE}๐Ÿ“ฆ Aggiornamento dipendenze Node.js...${NC}" -sudo -u $IDS_USER npm install - -if [ $? -eq 0 ]; then - echo -e "${GREEN}โœ… Dipendenze Node.js aggiornate${NC}" -else - echo -e "${YELLOW}โš ๏ธ Possibili warning npm (spesso ignorabili)${NC}" -fi - -# Aggiorna dipendenze Python -echo -e "\n${BLUE}๐Ÿ“ฆ Aggiornamento dipendenze Python...${NC}" -cd python_ml -sudo -u $IDS_USER /usr/bin/python3.11 -m pip install --upgrade -r requirements.txt -cd .. - -if [ $? -eq 0 ]; then - echo -e "${GREEN}โœ… Dipendenze Python aggiornate${NC}" -fi - -# Restart servizi -echo -e "\n${BLUE}๐Ÿ”„ Restart servizi...${NC}" -if [ -f "./deployment/restart_all.sh" ]; then - chmod +x ./deployment/restart_all.sh - ./deployment/restart_all.sh - echo -e "${GREEN}โœ… Servizi riavviati${NC}" -else - echo -e "${YELLOW}โš ๏ธ Script restart_all.sh non trovato${NC}" - echo -e "${YELLOW} Riavvia manualmente i servizi se necessario${NC}" -fi - -echo "" -echo -e "${GREEN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" -echo -e "${GREEN}โ•‘ โœ… AGGIORNAMENTO COMPLETATO โ•‘${NC}" -echo -e "${GREEN}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" -echo "" -echo -e "${BLUE}๐Ÿ“‹ VERIFICA SISTEMA:${NC}" -echo -e " โ€ข Log backend: ${YELLOW}tail -f /var/log/ids/backend.log${NC}" -echo -e " โ€ข Log frontend: ${YELLOW}tail -f /var/log/ids/frontend.log${NC}" -echo -e " โ€ข API backend: ${YELLOW}curl http://localhost:8000/health${NC}" -echo -e " โ€ข Frontend: ${YELLOW}curl http://localhost:5000${NC}" -echo "" -echo -e "${BLUE}๐Ÿ“Š STATO SERVIZI:${NC}" -ps aux | grep -E 'python.*main|npm.*dev|syslog_parser' | grep -v grep -echo "" - -exit 0