From c8efe5c942eb5679fc1d2396006141f9a74bd698 Mon Sep 17 00:00:00 2001 From: marco370 <48531002-marco370@users.noreply.replit.com> Date: Mon, 16 Feb 2026 11:29:12 +0000 Subject: [PATCH] Improve ML stats display and add database fallback Add backend logic for ML stats to use database when unavailable and update frontend to show offline status. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: intermediate_checkpoint Replit-Commit-Event-Id: b69b4d7d-5491-401d-a003-d99b33ae655d Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/MmMtYN7 --- client/src/pages/Training.tsx | 65 ++++++++++++---- replit.md | 2 +- server/routes.ts | 140 ++++++++++++++++++++++------------ 3 files changed, 140 insertions(+), 67 deletions(-) diff --git a/client/src/pages/Training.tsx b/client/src/pages/Training.tsx index 6571118..94e1a58 100644 --- a/client/src/pages/Training.tsx +++ b/client/src/pages/Training.tsx @@ -33,10 +33,13 @@ import { Input } from "@/components/ui/input"; import { Checkbox } from "@/components/ui/checkbox"; interface MLStatsResponse { + source?: string; + ml_backend_status?: string; logs?: { total: number; last_hour: number }; - detections?: { total: number; blocked: number }; + detections?: { total: number; blocked: number; critical?: number; unique_ips?: number }; routers?: { active: number }; latest_training?: any; + logs_24h?: number; } const trainFormSchema = z.object({ @@ -147,21 +150,43 @@ export default function TrainingPage() {

+ {/* ML Backend Status Warning */} + {mlStats?.ml_backend_status === "offline" && ( + + + +
+

ML Backend Python offline

+

+ Le statistiche mostrate provengono dal database. Training e detection manuali non sono disponibili fino al riavvio del servizio. +

+
+ + + )} + {/* ML Stats */} {mlStats && (
- Log Totali + + {mlStats.source === "database_fallback" ? "Log Ultime 24h" : "Log Totali"} +
- {mlStats.logs?.total?.toLocaleString() || 0} + {(mlStats.source === "database_fallback" + ? mlStats.logs_24h + : mlStats.logs?.total + )?.toLocaleString() || 0}
-

- Ultima ora: {mlStats.logs?.last_hour?.toLocaleString() || 0} -

+ {mlStats.source !== "database_fallback" && ( +

+ Ultima ora: {mlStats.logs?.last_hour?.toLocaleString() || 0} +

+ )} @@ -172,22 +197,30 @@ export default function TrainingPage() {
- {mlStats.detections?.total || 0} + {mlStats.detections?.total?.toLocaleString() || 0}

- Bloccati: {mlStats.detections?.blocked || 0} + Bloccati: {mlStats.detections?.blocked?.toLocaleString() || 0} + {mlStats.detections?.critical !== undefined && ( + | Critici: {mlStats.detections.critical.toLocaleString()} + )}

- Router Attivi + + {mlStats.source === "database_fallback" ? "IP Unici" : "Router Attivi"} +
- {mlStats.routers?.active || 0} + {(mlStats.source === "database_fallback" + ? mlStats.detections?.unique_ips + : mlStats.routers?.active + )?.toLocaleString() || 0}
@@ -214,9 +247,9 @@ export default function TrainingPage() {

- @@ -265,7 +298,7 @@ export default function TrainingPage() { > Annulla - @@ -294,9 +327,9 @@ export default function TrainingPage() {

- @@ -377,7 +410,7 @@ export default function TrainingPage() { > Annulla - diff --git a/replit.md b/replit.md index ee9b2b8..f5375c2 100644 --- a/replit.md +++ b/replit.md @@ -28,7 +28,7 @@ The IDS employs a React-based frontend for real-time monitoring, detection visua - **Automated Blocking**: Critical IPs (score >= 80) are automatically blocked in parallel across configured MikroTik routers via their REST API. **Auto-unblock on whitelist**: When an IP is added to the whitelist, it is automatically removed from all router blocklists. Manual unblock button available in Detections page. - **Public Lists Integration (v2.0.0 - CIDR Complete)**: Automatic fetcher syncs blacklist/whitelist feeds every 10 minutes (Spamhaus, Talos, AWS, GCP, Cloudflare, IANA, NTP Pool). **Full CIDR support** using PostgreSQL INET/CIDR types with `<<=` containment operators for network range matching. Priority-based merge logic: Manual whitelist > Public whitelist > Blacklist (CIDR-aware). Detections created for blacklisted IPs/ranges (excluding whitelisted ranges). CRUD API + UI for list management. See `deployment/docs/PUBLIC_LISTS_V2_CIDR.md` for implementation details. - **Automatic Cleanup**: An hourly systemd timer (`cleanup_detections.py`) removes old detections (48h) and auto-unblocks IPs (2h). -- **Service Monitoring & Management**: A dashboard provides real-time status (ML Backend, Database, Syslog Parser). API endpoints, secured with API key authentication and Systemd integration, allow for service management (start/stop/restart) of Python services. +- **Service Monitoring & Management**: A dashboard provides real-time status (ML Backend, Database, Syslog Parser, Analytics Aggregator). **Syslog Parser check is database-based** (counts logs in last 30 minutes) and independent of ML Backend availability. ML Stats endpoint has database fallback when Python backend is offline. Training UI shows offline warning and disables actions when ML Backend is unavailable. API endpoints, secured with API key authentication and Systemd integration, allow for service management (start/stop/restart) of Python services. - **IP Geolocation**: Integration with `ip-api.com` enriches detection data with geographical and AS information, utilizing intelligent caching. - **Database Management**: PostgreSQL is used for all persistent data. An intelligent database versioning system ensures efficient SQL migrations (v8 with forced INET/CIDR column types for network range matching). Migration 008 unconditionally recreates INET columns to fix type mismatches. Dual-mode database drivers (`@neondatabase/serverless` for Replit, `pg` for AlmaLinux) ensure environment compatibility. - **Microservices**: Clear separation of concerns between the Python ML backend and the Node.js API backend. diff --git a/server/routes.ts b/server/routes.ts index 4517d3a..ec09efa 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -1,7 +1,7 @@ import type { Express } from "express"; import { createServer, type Server } from "http"; import { storage } from "./storage"; -import { insertRouterSchema, insertDetectionSchema, insertWhitelistSchema, insertPublicListSchema, networkAnalytics, routers, detections, networkLogs } from "@shared/schema"; +import { insertRouterSchema, insertDetectionSchema, insertWhitelistSchema, insertPublicListSchema, networkAnalytics, routers, detections, networkLogs, trainingHistory } from "@shared/schema"; import { db } from "./db"; import { desc, eq, gte, sql } from "drizzle-orm"; import { blockIpOnAllRouters, unblockIpOnAllRouters, bulkBlockIps, testRouterConnection } from "./mikrotik"; @@ -627,7 +627,8 @@ export async function registerRoutes(app: Express): Promise { app.post("/api/ml/block-all-critical", async (req, res) => { try { - const { min_score = 80, list_name = "ddos_blocked" } = req.body; + const { min_score = 80, list_name = "ddos_blocked", limit = 100 } = req.body; + const maxIps = Math.min(Number(limit) || 100, 500); const allRouters = await storage.getAllRouters(); const enabledRouters = allRouters.filter(r => r.enabled); @@ -643,23 +644,35 @@ export async function registerRoutes(app: Express): Promise { AND blocked = false AND source_ip NOT IN (SELECT ip_address FROM whitelist WHERE active = true) GROUP BY source_ip - ORDER BY max_score DESC` + ORDER BY max_score DESC + LIMIT ${maxIps}` ); const rows = (unblockedDetections as any).rows || unblockedDetections; + const totalUnblockedResult = await db.execute( + sql`SELECT COUNT(DISTINCT source_ip) as count + FROM detections + WHERE CAST(risk_score AS FLOAT) >= ${min_score} + AND blocked = false + AND source_ip NOT IN (SELECT ip_address FROM whitelist WHERE active = true)` + ); + const totalUnblockedRows = (totalUnblockedResult as any).rows || totalUnblockedResult; + const totalUnblocked = parseInt(totalUnblockedRows[0]?.count || "0"); + if (!rows || rows.length === 0) { return res.json({ message: "Nessun IP critico da bloccare", blocked: 0, failed: 0, total_critical: 0, + remaining: 0, skipped: 0 }); } const ipList = rows.map((r: any) => r.source_ip); - console.log(`[BLOCK-ALL] Avvio blocco massivo: ${ipList.length} IP con score >= ${min_score} su ${enabledRouters.length} router`); + console.log(`[BLOCK-ALL] Avvio blocco massivo: ${ipList.length}/${totalUnblocked} IP con score >= ${min_score} su ${enabledRouters.length} router`); const result = await bulkBlockIps( enabledRouters as any, @@ -686,12 +699,15 @@ export async function registerRoutes(app: Express): Promise { console.log(`[BLOCK-ALL] Database aggiornato: ${blockedIps.length} IP marcati come bloccati`); } + const remaining = totalUnblocked - ipList.length; res.json({ - message: `Blocco massivo completato: ${result.blocked} IP bloccati, ${result.failed} falliti, ${result.skipped} già bloccati`, + message: `Blocco massivo completato: ${result.blocked} IP bloccati, ${result.failed} falliti, ${result.skipped} già bloccati` + + (remaining > 0 ? `. Rimangono ${remaining} IP da bloccare.` : ''), blocked: result.blocked, failed: result.failed, skipped: result.skipped, total_critical: ipList.length, + remaining, details: result.details.slice(0, 100) }); } catch (error: any) { @@ -703,7 +719,7 @@ export async function registerRoutes(app: Express): Promise { app.get("/api/ml/stats", async (req, res) => { try { const controller = new AbortController(); - const timeout = setTimeout(() => controller.abort(), 10000); // 10s timeout for stats + const timeout = setTimeout(() => controller.abort(), 5000); const response = await fetch(`${ML_BACKEND_URL}/stats`, { headers: getMLBackendHeaders(), @@ -713,23 +729,49 @@ export async function registerRoutes(app: Express): Promise { clearTimeout(timeout); if (!response.ok) { - const errorData = await response.json().catch(() => ({})); - return res.status(response.status).json({ - error: errorData.detail || "Failed to fetch ML stats", - status: response.status, - }); + throw new Error(`HTTP ${response.status}`); } const data = await response.json(); res.json(data); } catch (error: any) { - if (error.name === 'AbortError') { - return res.status(504).json({ error: "Stats timeout" }); + try { + const latestTraining = await db + .select() + .from(trainingHistory) + .orderBy(desc(trainingHistory.trainedAt)) + .limit(1); + + const detectionStats = await db.execute( + sql`SELECT + COUNT(*) as total_detections, + COUNT(*) FILTER (WHERE blocked = true) as blocked_count, + COUNT(*) FILTER (WHERE CAST(risk_score AS FLOAT) >= 80) as critical_count, + COUNT(DISTINCT source_ip) as unique_ips + FROM detections` + ); + const statsRows = (detectionStats as any).rows || detectionStats; + + const logCount = await db.execute( + sql`SELECT COUNT(*) as count FROM network_logs WHERE timestamp > NOW() - INTERVAL '24 hours'` + ); + const logRows = (logCount as any).rows || logCount; + + res.json({ + source: "database_fallback", + ml_backend_status: "offline", + latest_training: latestTraining[0] || null, + detections: { + total: parseInt(statsRows[0]?.total_detections || "0"), + blocked: parseInt(statsRows[0]?.blocked_count || "0"), + critical: parseInt(statsRows[0]?.critical_count || "0"), + unique_ips: parseInt(statsRows[0]?.unique_ips || "0"), + }, + logs_24h: parseInt(logRows[0]?.count || "0"), + }); + } catch (dbError: any) { + res.status(503).json({ error: "ML backend offline and database fallback failed" }); } - if (error.code === 'ECONNREFUSED') { - return res.status(503).json({ error: "ML backend not available" }); - } - res.status(500).json({ error: error.message || "Failed to fetch ML stats" }); } }); @@ -784,45 +826,43 @@ export async function registerRoutes(app: Express): Promise { services.database.details = { error: error.message }; } - // Check Python Services via authenticated endpoint + // Check Syslog Parser via database (independent of ML Backend) try { - const controller2 = new AbortController(); - const timeout2 = setTimeout(() => controller2.abort(), 5000); - - const servicesResponse = await fetch(`${ML_BACKEND_URL}/services/status`, { - headers: getMLBackendHeaders(), - signal: controller2.signal, - }); - - clearTimeout(timeout2); - - if (servicesResponse.ok) { - const servicesData = await servicesResponse.json(); - - // Update syslog parser status - const parserInfo = servicesData.services?.syslog_parser; - if (parserInfo) { - services.syslogParser.status = parserInfo.running ? "running" : "offline"; - services.syslogParser.healthy = parserInfo.running; - services.syslogParser.details = { - systemd_unit: parserInfo.systemd_unit, - pid: parserInfo.details?.pid, - error: parserInfo.error, - }; - } - } else if (servicesResponse.status === 403) { - services.syslogParser.status = "error"; - services.syslogParser.healthy = false; - services.syslogParser.details = { error: "Authentication failed" }; + const recentLogsResult = await db.execute( + sql`SELECT COUNT(*) as count, MAX(timestamp) as last_log + FROM network_logs + WHERE timestamp > NOW() - INTERVAL '30 minutes'` + ); + const logRows = (recentLogsResult as any).rows || recentLogsResult; + const recentLogCount = parseInt(logRows[0]?.count || "0"); + const lastLogTime = logRows[0]?.last_log; + + if (recentLogCount > 0) { + services.syslogParser.status = "running"; + services.syslogParser.healthy = true; + services.syslogParser.details = { + recentLogs30min: recentLogCount, + lastLog: lastLogTime, + }; } else { - throw new Error(`HTTP ${servicesResponse.status}`); + const lastLogEverResult = await db.execute( + sql`SELECT MAX(timestamp) as last_log FROM network_logs` + ); + const lastLogEverRows = (lastLogEverResult as any).rows || lastLogEverResult; + const lastLogEver = lastLogEverRows[0]?.last_log; + + services.syslogParser.status = "offline"; + services.syslogParser.healthy = false; + services.syslogParser.details = { + recentLogs30min: 0, + lastLog: lastLogEver || "Never", + warning: "No logs received in last 30 minutes", + }; } } catch (error: any) { services.syslogParser.status = "error"; services.syslogParser.healthy = false; - services.syslogParser.details = { - error: error.code === 'ECONNREFUSED' ? "ML Backend offline" : error.message - }; + services.syslogParser.details = { error: error.message }; } // Check Analytics Aggregator (via last record timestamp)