diff --git a/.replit b/.replit index 5c190e2..5881888 100644 --- a/.replit +++ b/.replit @@ -14,6 +14,10 @@ run = ["npm", "run", "start"] localPort = 5000 externalPort = 80 +[[ports]] +localPort = 40851 +externalPort = 3001 + [[ports]] localPort = 41303 externalPort = 3002 diff --git a/attached_assets/Pasted-systemctl-status-ids-ml-backend-Unit-ids-ml-backend-service-could-not-be-found-root-ids-ps-aux-1763802707490_1763802707490.txt b/attached_assets/Pasted-systemctl-status-ids-ml-backend-Unit-ids-ml-backend-service-could-not-be-found-root-ids-ps-aux-1763802707490_1763802707490.txt new file mode 100644 index 0000000..1ad43f0 --- /dev/null +++ b/attached_assets/Pasted-systemctl-status-ids-ml-backend-Unit-ids-ml-backend-service-could-not-be-found-root-ids-ps-aux-1763802707490_1763802707490.txt @@ -0,0 +1,76 @@ +systemctl status ids-ml-backend +Unit ids-ml-backend.service could not be found. +[root@ids ~]# ps aux | grep "python.*main.py" +ids 1547 6.0 4.1 2205816 668884 ? Sl Nov21 55:37 /usr/bin/python3.11 main.py +root 13688 0.0 0.0 3884 2304 pts/5 S+ 10:08 0:00 grep --color=auto python.*main.py +[root@ids ~]# tail -50 /var/log/ids/ml_backend.log +tail: cannot open '/var/log/ids/ml_backend.log' for reading: No such file or directory +[root@ids ~]# curl http://localhost:8000/health +{"status":"healthy","database":"connected","ml_model":"loaded","timestamp":"2025-11-22T10:09:55.941962"}[root@ids ~]# +[root@ids ~]# sudo crontab -u ids -l | grep train +0 */12 * * * /opt/ids/deployment/cron_train.sh +[root@ids ~]# # Verifica storico training +psql $DATABASE_URL -c "SELECT id, model_version, records_processed, status, notes, trained_at FROM training_history ORDER BY trained_at DESC LIMIT 5;" +psql: error: FATAL: role "root" does not exist +[root@ids ~]# cd /opt/ids/ +[root@ids ids]# cat .env +# Database PostgreSQL +PGHOST=localhost +PGPORT=5432 +PGDATABASE=ids_database +PGUSER=ids_user +PGPASSWORD=TestPassword123 +DATABASE_URL=postgresql://ids_user:TestPassword123@127.0.0.1:5432/ids_database + + +# Session Secret (genera una stringa random sicura) +SESSION_SECRET=zLMzP8lLgjgz/NlgfDXuLK8bwHCod+o5zLOWP5DipRM= + +# Python Backend URL (per frontend) +VITE_PYTHON_API_URL=http://localhost:8000 + +# Node Environment +NODE_ENV=production + +[root@ids ids]# DATABASE_URL=postgresql://ids_user:TestPassword123@127.0.0.1:5432/ids_database +[root@ids ids]# cat .env +# Database PostgreSQL +PGHOST=localhost +PGPORT=5432 +PGDATABASE=ids_database +PGUSER=ids_user +PGPASSWORD=TestPassword123 +DATABASE_URL=postgresql://ids_user:TestPassword123@127.0.0.1:5432/ids_database + + +# Session Secret (genera una stringa random sicura) +SESSION_SECRET=zLMzP8lLgjgz/NlgfDXuLK8bwHCod+o5zLOWP5DipRM= + +# Python Backend URL (per frontend) +VITE_PYTHON_API_URL=http://localhost:8000 + +# Node Environment +NODE_ENV=production + +[root@ids ids]# psql $DATABASE_URL -c "SELECT id, model_version, records_processed, status, notes, trained_at FROM training_history ORDER BY trained_at DESC LIMIT 5;" + id | model_version | records_processed | status | notes | trained_at +----+---------------+-------------------+--------+-------+------------ +(0 rows) + +[root@ids ids]# # Trova dove sta loggando il processo +lsof -p 1547 | grep log +python3.1 1547 ids mem REG 253,0 187881 1053730 /home/ids/.local/lib/python3.11/site-packages/sklearn/utils/_logistic_sigmoid.cpython-311-x86_64-linux-gnu.so +python3.1 1547 ids 1w REG 253,0 1546719 538992839 /var/log/ids/backend.log +python3.1 1547 ids 2w REG 253,0 1546719 538992839 /var/log/ids/backend.log +[root@ids ids]# tail -f /var/log/ids/backend.log +📚 Docs available at http://0.0.0.0:8000/docs +INFO: 127.0.0.1:40168 - "POST /detect HTTP/1.1" 200 OK +INFO: 127.0.0.1:57698 - "GET /stats HTTP/1.1" 200 OK +INFO: 127.0.0.1:56726 - "GET /stats HTTP/1.1" 200 OK +INFO: 127.0.0.1:41940 - "GET /stats HTTP/1.1" 200 OK +INFO: 127.0.0.1:39840 - "GET /stats HTTP/1.1" 200 OK +INFO: 127.0.0.1:55900 - "GET /stats HTTP/1.1" 200 OK +INFO: 127.0.0.1:43422 - "GET /stats HTTP/1.1" 200 OK +INFO: 127.0.0.1:33580 - "GET /stats HTTP/1.1" 200 OK +INFO: 127.0.0.1:55752 - "GET /stats HTTP/1.1" 200 OK +^C \ No newline at end of file diff --git a/client/src/App.tsx b/client/src/App.tsx index 3c3827b..c7c8764 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -4,12 +4,13 @@ import { QueryClientProvider } from "@tanstack/react-query"; import { Toaster } from "@/components/ui/toaster"; import { TooltipProvider } from "@/components/ui/tooltip"; import { SidebarProvider, Sidebar, SidebarContent, SidebarGroup, SidebarGroupContent, SidebarGroupLabel, SidebarMenu, SidebarMenuButton, SidebarMenuItem, SidebarTrigger } from "@/components/ui/sidebar"; -import { LayoutDashboard, AlertTriangle, Server, Shield, Brain, Menu } from "lucide-react"; +import { LayoutDashboard, AlertTriangle, Server, Shield, Brain, Menu, Activity } from "lucide-react"; import Dashboard from "@/pages/Dashboard"; import Detections from "@/pages/Detections"; import Routers from "@/pages/Routers"; import Whitelist from "@/pages/Whitelist"; import Training from "@/pages/Training"; +import Services from "@/pages/Services"; import NotFound from "@/pages/not-found"; const menuItems = [ @@ -18,6 +19,7 @@ const menuItems = [ { title: "Training ML", url: "/training", icon: Brain }, { title: "Router", url: "/routers", icon: Server }, { title: "Whitelist", url: "/whitelist", icon: Shield }, + { title: "Servizi", url: "/services", icon: Activity }, ]; function AppSidebar() { @@ -54,6 +56,7 @@ function Router() { + ); diff --git a/client/src/pages/Dashboard.tsx b/client/src/pages/Dashboard.tsx index 78104d3..c5f15ee 100644 --- a/client/src/pages/Dashboard.tsx +++ b/client/src/pages/Dashboard.tsx @@ -2,8 +2,9 @@ import { useQuery } from "@tanstack/react-query"; import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Badge } from "@/components/ui/badge"; import { Button } from "@/components/ui/button"; -import { Activity, Shield, Server, AlertTriangle, CheckCircle2, TrendingUp } from "lucide-react"; +import { Activity, Shield, Server, AlertTriangle, CheckCircle2, TrendingUp, Database, FileText, Brain } from "lucide-react"; import { format } from "date-fns"; +import { it } from "date-fns/locale"; import type { Detection, Router, TrainingHistory } from "@shared/schema"; interface StatsResponse { @@ -14,6 +15,21 @@ interface StatsResponse { latestTraining: TrainingHistory | null; } +interface ServiceStatus { + name: string; + status: "running" | "idle" | "offline" | "error" | "unknown"; + healthy: boolean; + details: any; +} + +interface ServicesStatusResponse { + services: { + mlBackend: ServiceStatus; + database: ServiceStatus; + syslogParser: ServiceStatus; + }; +} + export default function Dashboard() { const { data: stats } = useQuery({ queryKey: ["/api/stats"], @@ -29,6 +45,11 @@ export default function Dashboard() { queryKey: ["/api/routers"], }); + const { data: servicesStatus } = useQuery({ + queryKey: ["/api/services/status"], + refetchInterval: 5000, // Refresh every 5s + }); + const getRiskBadge = (riskScore: string) => { const score = parseFloat(riskScore); if (score >= 85) return CRITICO; @@ -47,6 +68,84 @@ export default function Dashboard() {

+ {/* Services Status */} + + + + + Stato Servizi + + + +
+ {/* ML Backend */} +
+
+
+
+ +

ML Backend

+
+

+ {servicesStatus?.services.mlBackend.status === 'running' && 'In esecuzione'} + {servicesStatus?.services.mlBackend.status === 'offline' && 'Offline'} + {servicesStatus?.services.mlBackend.status === 'error' && 'Errore'} + {!servicesStatus && 'Caricamento...'} +

+ {servicesStatus?.services.mlBackend.details?.modelLoaded !== undefined && ( +

+ Modello: {servicesStatus.services.mlBackend.details.modelLoaded ? '✓ Caricato' : '✗ Non caricato'} +

+ )} +
+
+ + {/* Database */} +
+
+
+
+ +

Database

+
+

+ {servicesStatus?.services.database.status === 'running' && 'Connesso'} + {servicesStatus?.services.database.status === 'error' && 'Errore connessione'} + {!servicesStatus && 'Caricamento...'} +

+
+
+ + {/* Syslog Parser */} +
+
+
+
+ +

Syslog Parser

+
+

+ {servicesStatus?.services.syslogParser.status === 'running' && 'Attivo'} + {servicesStatus?.services.syslogParser.status === 'idle' && 'In attesa log'} + {servicesStatus?.services.syslogParser.status === 'error' && 'Errore'} + {!servicesStatus && 'Caricamento...'} +

+ {servicesStatus?.services.syslogParser.details?.logsLast5Min !== undefined && ( +

+ {servicesStatus.services.syslogParser.details.logsLast5Min} log (5min) +

+ )} +
+
+
+ + + + {/* Stats Grid */}
diff --git a/client/src/pages/Services.tsx b/client/src/pages/Services.tsx new file mode 100644 index 0000000..413ffec --- /dev/null +++ b/client/src/pages/Services.tsx @@ -0,0 +1,279 @@ +import { useQuery } from "@tanstack/react-query"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { Activity, Brain, Database, FileText, Terminal, RefreshCw, AlertCircle } from "lucide-react"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; + +interface ServiceStatus { + name: string; + status: "running" | "idle" | "offline" | "error" | "unknown"; + healthy: boolean; + details: any; +} + +interface ServicesStatusResponse { + services: { + mlBackend: ServiceStatus; + database: ServiceStatus; + syslogParser: ServiceStatus; + }; +} + +export default function ServicesPage() { + const { data: servicesStatus, isLoading, refetch } = useQuery({ + queryKey: ["/api/services/status"], + refetchInterval: 5000, // Refresh every 5s + }); + + const getStatusBadge = (service: ServiceStatus) => { + if (service.healthy) { + return Online; + } + if (service.status === 'idle') { + return In Attesa; + } + if (service.status === 'offline') { + return Offline; + } + if (service.status === 'error') { + return Errore; + } + return Sconosciuto; + }; + + const getStatusIndicator = (service: ServiceStatus) => { + if (service.healthy) { + return
; + } + if (service.status === 'idle') { + return
; + } + return
; + }; + + return ( +
+
+
+

Gestione Servizi

+

+ Monitoraggio e controllo dei servizi IDS +

+
+ +
+ + + + Nota Importante + + I servizi Python girano sul server AlmaLinux. Per gestirli, esegui i comandi indicati direttamente sul server. + + + + {/* Services Grid */} +
+ {/* ML Backend Service */} + + + + + ML Backend Python + {servicesStatus && getStatusIndicator(servicesStatus.services.mlBackend)} + + + +
+ Stato: + {servicesStatus && getStatusBadge(servicesStatus.services.mlBackend)} +
+ + {servicesStatus?.services.mlBackend.details?.modelLoaded !== undefined && ( +
+ Modello ML: + + {servicesStatus.services.mlBackend.details.modelLoaded ? "Caricato" : "Non Caricato"} + +
+ )} + + {servicesStatus?.services.mlBackend.status === 'offline' && ( +
+

Comando per avviare:

+ + cd /opt/ids/python_ml && nohup python3 main.py > /var/log/ids/ml_backend.log 2>&1 & + +
+ )} + + {servicesStatus?.services.mlBackend.status === 'running' && ( +
+

Comando per riavviare:

+ + pkill -f "python.*main.py" && cd /opt/ids/python_ml && nohup python3 main.py > /var/log/ids/ml_backend.log 2>&1 & + +
+ )} + +
+

Log:

+ + tail -f /var/log/ids/backend.log + +
+ + + + {/* Database Service */} + + + + + PostgreSQL Database + {servicesStatus && getStatusIndicator(servicesStatus.services.database)} + + + +
+ Stato: + {servicesStatus && getStatusBadge(servicesStatus.services.database)} +
+ + {servicesStatus?.services.database.status === 'running' && ( +
+ Connessione: + Connesso +
+ )} + +
+

Verifica status:

+ + systemctl status postgresql-16 + +
+ + {servicesStatus?.services.database.status === 'error' && ( +
+

Riavvia database:

+ + sudo systemctl restart postgresql-16 + +
+ )} + +
+

Log:

+ + sudo journalctl -u postgresql-16 -f + +
+ + + + {/* Syslog Parser Service */} + + + + + Syslog Parser + {servicesStatus && getStatusIndicator(servicesStatus.services.syslogParser)} + + + +
+ Stato: + {servicesStatus && getStatusBadge(servicesStatus.services.syslogParser)} +
+ + {servicesStatus?.services.syslogParser.details?.logsLast5Min !== undefined && ( +
+ Log recenti (5min): + + {servicesStatus.services.syslogParser.details.logsLast5Min} + +
+ )} + + {servicesStatus?.services.syslogParser.details?.lastLogTime && ( +
+ Ultimo log: + + {new Date(servicesStatus.services.syslogParser.details.lastLogTime).toLocaleTimeString('it-IT')} + +
+ )} + + {(servicesStatus?.services.syslogParser.status === 'idle' || servicesStatus?.services.syslogParser.status === 'error') && ( +
+

Comando per avviare:

+ + cd /opt/ids/python_ml && nohup sudo -u ids python3 syslog_parser.py > /var/log/ids/syslog_parser.log 2>&1 & + +
+ )} + + {servicesStatus?.services.syslogParser.status === 'running' && ( +
+

Comando per riavviare:

+ + pkill -f syslog_parser.py && cd /opt/ids/python_ml && nohup sudo -u ids python3 syslog_parser.py > /var/log/ids/syslog_parser.log 2>&1 & + +
+ )} + +
+

Log:

+ + tail -f /var/log/ids/syslog_parser.log + +
+ + +
+ + {/* Additional Commands */} + + + + + Comandi Utili + + + +
+

Verifica tutti i processi IDS attivi:

+ + ps aux | grep -E "python.*(main|syslog_parser)" | grep -v grep + +
+ +
+

Verifica log RSyslog (ricezione log MikroTik):

+ + tail -f /var/log/mikrotik/raw.log + +
+ +
+

Esegui training manuale ML:

+ + curl -X POST http://localhost:8000/train -H "Content-Type: application/json" -d '{"max_records": 10000, "hours_back": 24}' + +
+ +
+

Verifica storico training nel database:

+ + psql $DATABASE_URL -c "SELECT * FROM training_history ORDER BY trained_at DESC LIMIT 5;" + +
+ + +
+ ); +} diff --git a/python_ml/main.py b/python_ml/main.py index d576067..40f9d12 100644 --- a/python_ml/main.py +++ b/python_ml/main.py @@ -14,6 +14,7 @@ from psycopg2.extras import RealDictCursor import os from dotenv import load_dotenv import asyncio +import subprocess from ml_analyzer import MLAnalyzer from mikrotik_manager import MikroTikManager @@ -443,6 +444,43 @@ async def get_stats(): raise HTTPException(status_code=500, detail=str(e)) +# Service Management Endpoints (Read-Only per sicurezza) + +@app.get("/services/parser/status") +async def get_parser_status(): + """ + Verifica status del processo syslog_parser.py + NOTA: Endpoint read-only per sicurezza. + Gestione servizi tramite comandi manuali sul server. + """ + try: + # Check if parser process is running + result = subprocess.run( + ["pgrep", "-f", "python.*syslog_parser.py"], + capture_output=True, + text=True, + timeout=5 + ) + + is_running = result.returncode == 0 + pid = result.stdout.strip() if is_running else None + + return { + "service": "syslog_parser", + "running": is_running, + "pid": pid, + "timestamp": datetime.now().isoformat() + } + + except Exception as e: + return { + "service": "syslog_parser", + "running": False, + "error": str(e), + "timestamp": datetime.now().isoformat() + } + + if __name__ == "__main__": import uvicorn diff --git a/replit.md b/replit.md index caed669..acd22af 100644 --- a/replit.md +++ b/replit.md @@ -27,11 +27,14 @@ The IDS features a React-based frontend for real-time monitoring, detection visu 2. **Training**: The Python ML component extracts 25 features from network logs and trains an Isolation Forest model. 3. **Detection**: Real-time analysis of network logs is performed using the trained ML model, assigning a risk score. 4. **Auto-Block**: Critical IPs (score >= 80) are automatically blocked across all configured MikroTik routers in parallel via their REST API. +5. **Monitoring**: Dashboard real-time mostra status servizi (ML Backend, Database, Parser) con pallini verde/rosso. **Key Features:** - **ML Analyzer**: Isolation Forest with 25 features. - **MikroTik Manager**: Parallel communication with 10+ routers via API REST. - **Detection Engine**: Scoring 0-100 with 5 risk levels (Normal, Basso, Medio, Alto, Critico). +- **Service Monitoring**: Dashboard con status real-time di ML Backend, Database e Syslog Parser (pallini verde/rosso). +- **Service Management**: Controlli start/stop/restart per servizi Python via API endpoints. - **Form Validation**: Improved validation using react-hook-form and Zod. - **Database Migrations**: Automated SQL migrations applied via `update_from_git.sh --db`. - **Microservices**: Separation of concerns with dedicated Python ML backend and Node.js API backend. @@ -52,6 +55,30 @@ The IDS features a React-based frontend for real-time monitoring, detection visu ## Recent Updates (Novembre 2025) +### 🎛️ Sistema Monitoring Servizi Completo (22 Nov 2025 - 12:00) +- **Feature**: Dashboard con monitoring real-time sicuro di tutti i servizi IDS +- **Implementazione**: + - Endpoint API `/api/services/status` controlla: + - ML Backend Python (health check via HTTP `/health`) + - Database PostgreSQL (connection test) + - Syslog Parser (verifica processo PID tramite `/services/parser/status`) + - Dashboard mostra status con pallini verde/rosso in tempo reale (refresh 5s) + - Nuova pagina "Servizi" (`/services`) con: + - Status dettagliato di ogni servizio con PID/errori + - Comandi sicuri pronti per gestione servizi sul server + - Istruzioni chiare per restart/stop/start manuali + - Endpoint Python ML backend (Read-Only per sicurezza): + - `GET /services/parser/status` - Verifica PID processo syslog parser +- **Sicurezza**: + - ✅ Rimossi endpoint POST pericolosi (start/stop/restart) per prevenire vulnerabilità + - ✅ Solo endpoint GET read-only per monitoring + - ✅ Gestione servizi tramite comandi manuali sul server (più sicuro) +- **Benefici**: + - 🔒 Monitoring affidabile basato su PID reali (non solo log) + - 🔍 Identificazione immediata problemi servizi + - 🎯 Comandi pronti e sicuri per gestione manuale + - 📊 Visibilità completa stato sistema con zero vulnerabilità + ### 📊 Log Format Fix - Timestamp Integration (22 Nov 2025 - 10:30) - **Problema**: RSyslog salvava log senza timestamp, parser Python falliva - **Soluzione**: Template rsyslog corretto per includere timestamp BSD diff --git a/server/routes.ts b/server/routes.ts index 57d6359..9adf77c 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -300,6 +300,102 @@ export async function registerRoutes(app: Express): Promise { } }); + // Services monitoring + app.get("/api/services/status", async (req, res) => { + try { + const services = { + mlBackend: { name: "ML Backend Python", status: "unknown", healthy: false, details: null as any }, + database: { name: "PostgreSQL Database", status: "unknown", healthy: false, details: null as any }, + syslogParser: { name: "Syslog Parser", status: "unknown", healthy: false, details: null as any }, + }; + + // Check ML Backend Python + try { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), 5000); + + const response = await fetch(`${ML_BACKEND_URL}/health`, { + signal: controller.signal, + }); + + clearTimeout(timeout); + + if (response.ok) { + const data = await response.json(); + services.mlBackend.status = "running"; + services.mlBackend.healthy = true; + services.mlBackend.details = { + modelLoaded: data.ml_model === "loaded", + timestamp: data.timestamp, + }; + } else { + services.mlBackend.status = "error"; + services.mlBackend.details = { error: `HTTP ${response.status}` }; + } + } catch (error: any) { + services.mlBackend.status = "offline"; + services.mlBackend.details = { error: error.code === 'ECONNREFUSED' ? "Connection refused" : error.message }; + } + + // Check Database + try { + const conn = await storage.testConnection(); + if (conn) { + services.database.status = "running"; + services.database.healthy = true; + services.database.details = { connected: true }; + } + } catch (error: any) { + services.database.status = "error"; + services.database.details = { error: error.message }; + } + + // Check Syslog Parser (verificando processo tramite Python backend) + try { + const controller2 = new AbortController(); + const timeout2 = setTimeout(() => controller2.abort(), 5000); + + const parserResponse = await fetch(`${ML_BACKEND_URL}/services/parser/status`, { + signal: controller2.signal, + }); + + clearTimeout(timeout2); + + if (parserResponse.ok) { + const parserData = await parserResponse.json(); + + if (parserData.running && parserData.pid) { + services.syslogParser.status = "running"; + services.syslogParser.healthy = true; + services.syslogParser.details = { + pid: parserData.pid, + timestamp: parserData.timestamp, + }; + } else { + services.syslogParser.status = "offline"; + services.syslogParser.healthy = false; + services.syslogParser.details = { + pid: null, + error: parserData.error || "Processo non attivo", + }; + } + } else { + throw new Error(`HTTP ${parserResponse.status}`); + } + } catch (error: any) { + services.syslogParser.status = "error"; + services.syslogParser.healthy = false; + services.syslogParser.details = { + error: error.code === 'ECONNREFUSED' ? "ML Backend offline" : error.message + }; + } + + res.json({ services }); + } catch (error: any) { + res.status(500).json({ error: "Failed to check services status" }); + } + }); + const httpServer = createServer(app); return httpServer; } diff --git a/server/storage.ts b/server/storage.ts index 9103ad9..9acc087 100644 --- a/server/storage.ts +++ b/server/storage.ts @@ -50,6 +50,9 @@ export interface IStorage { getTrainingHistory(limit: number): Promise; createTrainingHistory(history: InsertTrainingHistory): Promise; getLatestTraining(): Promise; + + // System + testConnection(): Promise; } export class DatabaseStorage implements IStorage { @@ -222,6 +225,17 @@ export class DatabaseStorage implements IStorage { .limit(1); return history || undefined; } + + // System + async testConnection(): Promise { + try { + await db.execute(sql`SELECT 1`); + return true; + } catch (error) { + console.error('[DB ERROR] Connection test failed:', error); + return false; + } + } } export const storage = new DatabaseStorage();