From cbd03d9e64562b3a39fa24335f1eca39da1f69ae Mon Sep 17 00:00:00 2001 From: marco370 <48531002-marco370@users.noreply.replit.com> Date: Sat, 22 Nov 2025 11:34:36 +0000 Subject: [PATCH] Add network analytics and live dashboard features Introduce new network analytics capabilities with persistent storage, hourly and daily aggregations, and enhanced frontend visualizations. This includes API endpoints for retrieving analytics data, systemd services for automated aggregation, and UI updates for live and historical dashboards. Additionally, country flag emojis are now displayed on the detections page. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: intermediate_checkpoint Replit-Commit-Event-Id: 3c14f651-7633-4128-8526-314b4942b3a0 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/oGXAoP7 --- client/src/App.tsx | 10 +- client/src/lib/country-flags.ts | 62 ++++ client/src/pages/AnalyticsHistory.tsx | 320 ++++++++++++++++++ client/src/pages/DashboardLive.tsx | 308 +++++++++++++++++ client/src/pages/Detections.tsx | 10 +- .../005_create_network_analytics.sql | 48 +++ deployment/ids-analytics-aggregator.service | 21 ++ deployment/ids-analytics-aggregator.timer | 14 + deployment/setup_analytics_timer.sh | 63 ++++ python_ml/analytics_aggregator.py | 318 +++++++++++++++++ replit.md | 24 +- server/routes.ts | 31 ++ server/storage.ts | 33 +- shared/schema.ts | 44 ++- 14 files changed, 1300 insertions(+), 6 deletions(-) create mode 100644 client/src/lib/country-flags.ts create mode 100644 client/src/pages/AnalyticsHistory.tsx create mode 100644 client/src/pages/DashboardLive.tsx create mode 100644 database-schema/migrations/005_create_network_analytics.sql create mode 100644 deployment/ids-analytics-aggregator.service create mode 100644 deployment/ids-analytics-aggregator.timer create mode 100755 deployment/setup_analytics_timer.sh create mode 100644 python_ml/analytics_aggregator.py diff --git a/client/src/App.tsx b/client/src/App.tsx index c7c8764..4b4cdb5 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -4,9 +4,11 @@ import { QueryClientProvider } from "@tanstack/react-query"; import { Toaster } from "@/components/ui/toaster"; import { TooltipProvider } from "@/components/ui/tooltip"; import { SidebarProvider, Sidebar, SidebarContent, SidebarGroup, SidebarGroupContent, SidebarGroupLabel, SidebarMenu, SidebarMenuButton, SidebarMenuItem, SidebarTrigger } from "@/components/ui/sidebar"; -import { LayoutDashboard, AlertTriangle, Server, Shield, Brain, Menu, Activity } from "lucide-react"; +import { LayoutDashboard, AlertTriangle, Server, Shield, Brain, Menu, Activity, BarChart3, TrendingUp } from "lucide-react"; import Dashboard from "@/pages/Dashboard"; import Detections from "@/pages/Detections"; +import DashboardLive from "@/pages/DashboardLive"; +import AnalyticsHistory from "@/pages/AnalyticsHistory"; import Routers from "@/pages/Routers"; import Whitelist from "@/pages/Whitelist"; import Training from "@/pages/Training"; @@ -16,10 +18,12 @@ import NotFound from "@/pages/not-found"; const menuItems = [ { title: "Dashboard", url: "/", icon: LayoutDashboard }, { title: "Rilevamenti", url: "/detections", icon: AlertTriangle }, + { title: "Dashboard Live", url: "/dashboard-live", icon: Activity }, + { title: "Analytics Storici", url: "/analytics", icon: BarChart3 }, { title: "Training ML", url: "/training", icon: Brain }, { title: "Router", url: "/routers", icon: Server }, { title: "Whitelist", url: "/whitelist", icon: Shield }, - { title: "Servizi", url: "/services", icon: Activity }, + { title: "Servizi", url: "/services", icon: TrendingUp }, ]; function AppSidebar() { @@ -53,6 +57,8 @@ function Router() { + + diff --git a/client/src/lib/country-flags.ts b/client/src/lib/country-flags.ts new file mode 100644 index 0000000..a1b1616 --- /dev/null +++ b/client/src/lib/country-flags.ts @@ -0,0 +1,62 @@ +/** + * Country Flags Utilities + * Converte country code in flag emoji + */ + +/** + * Converte country code ISO 3166-1 alpha-2 in flag emoji + * Es: "IT" => "๐Ÿ‡ฎ๐Ÿ‡น", "US" => "๐Ÿ‡บ๐Ÿ‡ธ" + */ +export function getFlagEmoji(countryCode: string | null | undefined): string { + if (!countryCode || countryCode.length !== 2) { + return '๐Ÿณ๏ธ'; // Flag bianca per unknown + } + + const codePoints = countryCode + .toUpperCase() + .split('') + .map(char => 127397 + char.charCodeAt(0)); + + return String.fromCodePoint(...codePoints); +} + +/** + * Mappa nomi paesi comuni (fallback se API non ritorna country code) + */ +export const COUNTRY_CODE_MAP: Record = { + 'Italy': 'IT', + 'United States': 'US', + 'Russia': 'RU', + 'China': 'CN', + 'Germany': 'DE', + 'France': 'FR', + 'United Kingdom': 'GB', + 'Spain': 'ES', + 'Brazil': 'BR', + 'Japan': 'JP', + 'India': 'IN', + 'Canada': 'CA', + 'Australia': 'AU', + 'Netherlands': 'NL', + 'Switzerland': 'CH', + 'Sweden': 'SE', + 'Poland': 'PL', + 'Ukraine': 'UA', + 'Romania': 'RO', + 'Belgium': 'BE', +}; + +/** + * Ottieni flag da nome paese o country code + */ +export function getFlag(country: string | null | undefined, countryCode?: string | null): string { + if (countryCode) { + return getFlagEmoji(countryCode); + } + + if (country && COUNTRY_CODE_MAP[country]) { + return getFlagEmoji(COUNTRY_CODE_MAP[country]); + } + + return '๐Ÿณ๏ธ'; +} diff --git a/client/src/pages/AnalyticsHistory.tsx b/client/src/pages/AnalyticsHistory.tsx new file mode 100644 index 0000000..0dd4edd --- /dev/null +++ b/client/src/pages/AnalyticsHistory.tsx @@ -0,0 +1,320 @@ +import { useQuery } from "@tanstack/react-query"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + LineChart, Line, BarChart, Bar, AreaChart, Area, + XAxis, YAxis, CartesianGrid, Tooltip, Legend, ResponsiveContainer +} from "recharts"; +import { Calendar, TrendingUp, BarChart3, Globe, Download } from "lucide-react"; +import type { NetworkAnalytics } from "@shared/schema"; +import { format, parseISO } from "date-fns"; +import { useState } from "react"; + +export default function AnalyticsHistory() { + const [days, setDays] = useState(30); + + // Fetch historical analytics (daily aggregations) + const { data: analytics = [], isLoading } = useQuery({ + queryKey: ["/api/analytics/recent", { days, hourly: false }], + refetchInterval: 60000, // Aggiorna ogni minuto + }); + + // Prepara dati per grafici + const trendData = analytics + .map(a => { + // Parse JSON fields safely + let attacksByCountry = {}; + let attacksByType = {}; + + try { + attacksByCountry = a.attacksByCountry ? JSON.parse(a.attacksByCountry) : {}; + } catch {} + + try { + attacksByType = a.attacksByType ? JSON.parse(a.attacksByType) : {}; + } catch {} + + return { + date: format(new Date(a.date), "dd/MM"), + fullDate: a.date, + totalPackets: a.totalPackets || 0, + normalPackets: a.normalPackets || 0, + attackPackets: a.attackPackets || 0, + attackPercentage: a.totalPackets > 0 + ? ((a.attackPackets || 0) / a.totalPackets * 100).toFixed(1) + : "0", + uniqueIps: a.uniqueIps || 0, + attackUniqueIps: a.attackUniqueIps || 0, + }; + }) + .sort((a, b) => new Date(a.fullDate).getTime() - new Date(b.fullDate).getTime()); + + // Aggrega dati per paese (da tutti i giorni) + const countryAggregation: Record = {}; + analytics.forEach(a => { + if (a.attacksByCountry) { + try { + const countries = JSON.parse(a.attacksByCountry); + if (countries && typeof countries === 'object') { + Object.entries(countries).forEach(([country, count]) => { + if (typeof count === 'number') { + countryAggregation[country] = (countryAggregation[country] || 0) + count; + } + }); + } + } catch (e) { + console.warn('Failed to parse attacksByCountry:', e); + } + } + }); + + const topCountries = Object.entries(countryAggregation) + .map(([name, attacks]) => ({ name, attacks })) + .sort((a, b) => b.attacks - a.attacks) + .slice(0, 10); + + // Calcola metriche totali + const totalTraffic = analytics.reduce((sum, a) => sum + (a.totalPackets || 0), 0); + const totalAttacks = analytics.reduce((sum, a) => sum + (a.attackPackets || 0), 0); + const totalNormal = analytics.reduce((sum, a) => sum + (a.normalPackets || 0), 0); + const avgAttackRate = totalTraffic > 0 ? ((totalAttacks / totalTraffic) * 100).toFixed(2) : "0"; + + return ( +
+ {/* Header */} +
+
+

+ + Analytics Storici +

+

+ Statistiche permanenti per analisi long-term +

+
+ + {/* Time Range Selector */} +
+ + + +
+
+ + {isLoading && ( +
+ Caricamento dati storici... +
+ )} + + {!isLoading && analytics.length === 0 && ( + + + +

Nessun dato storico disponibile

+

+ I dati verranno aggregati automaticamente ogni ora dal sistema +

+ + + )} + + {!isLoading && analytics.length > 0 && ( + <> + {/* Summary KPIs */} +
+ + + + Traffico Totale ({days}g) + + + +
+ {totalTraffic.toLocaleString()} +
+

pacchetti

+ + + + + + + Traffico Normale + + + +
+ {totalNormal.toLocaleString()} +
+

+ {(100 - parseFloat(avgAttackRate)).toFixed(1)}% del totale +

+ + + + + + + Attacchi Totali + + + +
+ {totalAttacks.toLocaleString()} +
+

+ {avgAttackRate}% del traffico +

+ + + + + + + Media Giornaliera + + + +
+ {Math.round(totalTraffic / analytics.length).toLocaleString()} +
+

pacchetti/giorno

+ + +
+ + {/* Trend Line Chart */} + + + + + Trend Traffico (Normale + Attacchi) + + + + + + + + + + + + + + + + + + {/* Attack Rate Trend */} + + + Percentuale Attacchi nel Tempo + + + + + + + + + + + + + + + + {/* Top Countries (Historical) */} + + + + + Top 10 Paesi Attaccanti (Storico) + + + + {topCountries.length > 0 ? ( + + + + + + + + + + + ) : ( +
+ Nessun dato disponibile +
+ )} + + + + {/* Export Button (Placeholder) */} + + +
+
+

Export Report

+

+ Esporta i dati in formato CSV per analisi esterne +

+
+ +
+ + + + )} +
+ ); +} diff --git a/client/src/pages/DashboardLive.tsx b/client/src/pages/DashboardLive.tsx new file mode 100644 index 0000000..2d61a69 --- /dev/null +++ b/client/src/pages/DashboardLive.tsx @@ -0,0 +1,308 @@ +import { useQuery } from "@tanstack/react-query"; +import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; +import { Badge } from "@/components/ui/badge"; +import { Activity, Globe, Shield, TrendingUp, AlertTriangle } from "lucide-react"; +import { AreaChart, Area, BarChart, Bar, PieChart, Pie, Cell, XAxis, YAxis, CartesianGrid, Tooltip, Legend, ResponsiveContainer } from "recharts"; +import type { Detection, NetworkLog } from "@shared/schema"; +import { getFlag } from "@/lib/country-flags"; +import { format } from "date-fns"; + +export default function DashboardLive() { + // Dati ultimi 3 giorni + const { data: detections = [], isLoading: loadingDetections } = useQuery({ + queryKey: ["/api/detections"], + refetchInterval: 10000, // Aggiorna ogni 10s + }); + + const { data: logs = [], isLoading: loadingLogs } = useQuery({ + queryKey: ["/api/logs", { limit: 5000 }], + refetchInterval: 10000, + }); + + const isLoading = loadingDetections || loadingLogs; + + // Calcola metriche + const totalTraffic = logs.length; + const totalAttacks = detections.length; + const attackPercentage = totalTraffic > 0 ? ((totalAttacks / totalTraffic) * 100).toFixed(2) : "0"; + + const normalTraffic = totalTraffic - totalAttacks; + const blockedAttacks = detections.filter(d => d.blocked).length; + + // Aggrega traffico per paese + const trafficByCountry: Record = {}; + + detections.forEach(det => { + const country = det.country || "Unknown"; + if (!trafficByCountry[country]) { + trafficByCountry[country] = { + normal: 0, + attacks: 0, + flag: getFlag(det.country, det.countryCode) + }; + } + trafficByCountry[country].attacks++; + }); + + const countryChartData = Object.entries(trafficByCountry) + .map(([name, data]) => ({ + name: `${data.flag} ${name}`, + attacks: data.attacks, + normal: data.normal, + })) + .sort((a, b) => b.attacks - a.attacks) + .slice(0, 10); + + // Aggrega attacchi per tipo + const attacksByType: Record = {}; + detections.forEach(det => { + attacksByType[det.anomalyType] = (attacksByType[det.anomalyType] || 0) + 1; + }); + + const typeChartData = Object.entries(attacksByType).map(([name, value]) => ({ + name: name.replace('_', ' ').toUpperCase(), + value, + })); + + // Traffico normale vs attacchi (gauge data) + const trafficDistribution = [ + { name: 'Normal', value: normalTraffic, color: '#22c55e' }, + { name: 'Attacks', value: totalAttacks, color: '#ef4444' }, + ]; + + // Ultimi eventi (stream) + const recentEvents = [...detections] + .sort((a, b) => new Date(b.detectedAt).getTime() - new Date(a.detectedAt).getTime()) + .slice(0, 20); + + const COLORS = ['#ef4444', '#f97316', '#f59e0b', '#eab308', '#84cc16']; + + return ( +
+ {/* Header */} +
+

+ + Dashboard Live +

+

+ Monitoraggio real-time (ultimi 3 giorni) +

+
+ + {isLoading && ( +
+ Caricamento dati... +
+ )} + + {!isLoading && ( + <> + {/* KPI Cards */} +
+ + + + Traffico Totale + + + +
+ {totalTraffic.toLocaleString()} +
+

pacchetti

+ + + + + + + Traffico Normale + + + +
+ {normalTraffic.toLocaleString()} +
+

+ {(100 - parseFloat(attackPercentage)).toFixed(1)}% del totale +

+ + + + + + + Attacchi Rilevati + + + +
+ {totalAttacks} +
+

+ {attackPercentage}% del traffico +

+ + + + + + + IP Bloccati + + + +
+ {blockedAttacks} +
+

+ {totalAttacks > 0 ? ((blockedAttacks / totalAttacks) * 100).toFixed(1) : 0}% degli attacchi +

+ + +
+ + {/* Charts Row 1 */} +
+ {/* Traffic Distribution (Pie) */} + + + + + Distribuzione Traffico + + + + + + `${entry.name}: ${entry.value}`} + outerRadius={100} + fill="#8884d8" + dataKey="value" + > + {trafficDistribution.map((entry, index) => ( + + ))} + + + + + + + + + {/* Attacks by Type (Pie) */} + + + + + Tipi di Attacco + + + + {typeChartData.length > 0 ? ( + + + `${entry.name}: ${entry.value}`} + outerRadius={100} + fill="#8884d8" + dataKey="value" + > + {typeChartData.map((entry, index) => ( + + ))} + + + + + + ) : ( +
+ Nessun attacco rilevato +
+ )} + + +
+ + {/* Top Countries (Bar Chart) */} + + + + + Top 10 Paesi Attaccanti + + + + {countryChartData.length > 0 ? ( + + + + + + + + + + + ) : ( +
+ Nessun dato disponibile +
+ )} + + + + {/* Real-time Event Stream */} + + + + + Stream Eventi Recenti + + + +
+ {recentEvents.map(event => ( +
+
+ {event.countryCode && ( + + {getFlag(event.country, event.countryCode)} + + )} +
+ {event.sourceIp} +

+ {event.anomalyType.replace('_', ' ')} โ€ข {format(new Date(event.detectedAt), "HH:mm:ss")} +

+
+
+ + {event.blocked ? "Bloccato" : "Attivo"} + +
+ ))} +
+ + + + )} +
+ ); +} diff --git a/client/src/pages/Detections.tsx b/client/src/pages/Detections.tsx index 1eed712..2cbe173 100644 --- a/client/src/pages/Detections.tsx +++ b/client/src/pages/Detections.tsx @@ -7,6 +7,7 @@ import { AlertTriangle, Search, Shield, Eye, Globe, MapPin, Building2 } from "lu import { format } from "date-fns"; import { useState } from "react"; import type { Detection } from "@shared/schema"; +import { getFlag } from "@/lib/country-flags"; export default function Detections() { const [searchQuery, setSearchQuery] = useState(""); @@ -93,7 +94,14 @@ export default function Detections() { >
-
+
+ {/* Flag Emoji */} + {detection.countryCode && ( + + {getFlag(detection.country, detection.countryCode)} + + )} + {detection.sourceIp} diff --git a/database-schema/migrations/005_create_network_analytics.sql b/database-schema/migrations/005_create_network_analytics.sql new file mode 100644 index 0000000..1708f86 --- /dev/null +++ b/database-schema/migrations/005_create_network_analytics.sql @@ -0,0 +1,48 @@ +-- Migration 005: Create network_analytics table for permanent traffic statistics +-- This table stores aggregated traffic data (normal + attacks) with hourly and daily granularity +-- Data persists beyond the 3-day log retention for long-term analytics + +CREATE TABLE IF NOT EXISTS network_analytics ( + id VARCHAR PRIMARY KEY DEFAULT gen_random_uuid(), + date TIMESTAMP NOT NULL, + hour INT, -- NULL = daily aggregation, 0-23 = hourly + + -- Total traffic metrics + total_packets INT NOT NULL DEFAULT 0, + total_bytes BIGINT NOT NULL DEFAULT 0, + unique_ips INT NOT NULL DEFAULT 0, + + -- Normal traffic (non-anomalous) + normal_packets INT NOT NULL DEFAULT 0, + normal_bytes BIGINT NOT NULL DEFAULT 0, + normal_unique_ips INT NOT NULL DEFAULT 0, + top_normal_ips TEXT, -- JSON: [{ip, packets, bytes, country}] + + -- Attack/Anomaly traffic + attack_packets INT NOT NULL DEFAULT 0, + attack_bytes BIGINT NOT NULL DEFAULT 0, + attack_unique_ips INT NOT NULL DEFAULT 0, + attacks_by_country TEXT, -- JSON: {IT: 5, RU: 30, ...} + attacks_by_type TEXT, -- JSON: {ddos: 10, port_scan: 5, ...} + top_attackers TEXT, -- JSON: [{ip, country, risk_score, packets}] + + -- Geographic distribution (all traffic) + traffic_by_country TEXT, -- JSON: {IT: {normal: 100, attacks: 5}, ...} + + created_at TIMESTAMP NOT NULL DEFAULT NOW(), + + -- Ensure unique aggregation per date/hour + UNIQUE(date, hour) +); + +-- Indexes for fast queries +CREATE INDEX IF NOT EXISTS network_analytics_date_hour_idx ON network_analytics(date, hour); +CREATE INDEX IF NOT EXISTS network_analytics_date_idx ON network_analytics(date); + +-- Update schema version +INSERT INTO schema_version (version, description) +VALUES (5, 'Create network_analytics table for traffic statistics') +ON CONFLICT (id) DO UPDATE SET + version = 5, + description = 'Create network_analytics table for traffic statistics', + applied_at = NOW(); diff --git a/deployment/ids-analytics-aggregator.service b/deployment/ids-analytics-aggregator.service new file mode 100644 index 0000000..1c24c55 --- /dev/null +++ b/deployment/ids-analytics-aggregator.service @@ -0,0 +1,21 @@ +[Unit] +Description=IDS Analytics Aggregator - Hourly Traffic Statistics +After=network.target postgresql.service + +[Service] +Type=oneshot +User=ids +Group=ids +WorkingDirectory=/opt/ids/python_ml +EnvironmentFile=-/opt/ids/.env + +# Execute hourly aggregation +ExecStart=/opt/ids/venv/bin/python3 /opt/ids/python_ml/analytics_aggregator.py hourly + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ids-analytics + +[Install] +WantedBy=multi-user.target diff --git a/deployment/ids-analytics-aggregator.timer b/deployment/ids-analytics-aggregator.timer new file mode 100644 index 0000000..a0f553d --- /dev/null +++ b/deployment/ids-analytics-aggregator.timer @@ -0,0 +1,14 @@ +[Unit] +Description=IDS Analytics Aggregation Timer - Runs every hour +Requires=ids-analytics-aggregator.service + +[Timer] +# Run 5 minutes after the hour (e.g., 10:05, 11:05, 12:05) +# This gives time for logs to be collected +OnCalendar=*:05:00 + +# Run immediately if we missed a scheduled run +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/deployment/setup_analytics_timer.sh b/deployment/setup_analytics_timer.sh new file mode 100755 index 0000000..2b30508 --- /dev/null +++ b/deployment/setup_analytics_timer.sh @@ -0,0 +1,63 @@ +#!/bin/bash +# Setup systemd timer for analytics aggregation +# Deve essere eseguito come root + +set -e + +# Colors +RED='\033[0;31m' +GREEN='\033[0;32m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +NC='\033[0m' # No Color + +echo -e "${BLUE}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" +echo -e "${BLUE}โ•‘ IDS Analytics Timer Setup โ•‘${NC}" +echo -e "${BLUE}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" +echo "" + +# Check root +if [ "$EUID" -ne 0 ]; then + echo -e "${RED}โŒ Questo script deve essere eseguito come root${NC}" + echo -e "${YELLOW} Usa: sudo $0${NC}" + exit 1 +fi + +IDS_DIR="/opt/ids" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# Copy systemd files +echo -e "${BLUE}๐Ÿ“‹ Copia file systemd...${NC}" +cp "${SCRIPT_DIR}/ids-analytics-aggregator.service" /etc/systemd/system/ +cp "${SCRIPT_DIR}/ids-analytics-aggregator.timer" /etc/systemd/system/ + +# Set permissions +chmod 644 /etc/systemd/system/ids-analytics-aggregator.service +chmod 644 /etc/systemd/system/ids-analytics-aggregator.timer + +# Reload systemd +echo -e "${BLUE}๐Ÿ”„ Reload systemd daemon...${NC}" +systemctl daemon-reload + +# Enable and start timer +echo -e "${BLUE}โš™๏ธ Enable e start timer...${NC}" +systemctl enable ids-analytics-aggregator.timer +systemctl start ids-analytics-aggregator.timer + +# Check status +echo -e "\n${BLUE}๐Ÿ“Š Stato timer:${NC}" +systemctl status ids-analytics-aggregator.timer --no-pager + +echo -e "\n${BLUE}๐Ÿ“… Prossime esecuzioni:${NC}" +systemctl list-timers ids-analytics-aggregator.timer --no-pager + +echo -e "\n${GREEN}โ•”โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•—${NC}" +echo -e "${GREEN}โ•‘ โœ… ANALYTICS TIMER CONFIGURATO โ•‘${NC}" +echo -e "${GREEN}โ•šโ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•โ•${NC}" +echo "" +echo -e "${BLUE}๐Ÿ“ Comandi utili:${NC}" +echo -e " ${YELLOW}Stato timer:${NC} sudo systemctl status ids-analytics-aggregator.timer" +echo -e " ${YELLOW}Prossime run:${NC} sudo systemctl list-timers" +echo -e " ${YELLOW}Log aggregazione:${NC} sudo journalctl -u ids-analytics-aggregator -f" +echo -e " ${YELLOW}Test manuale:${NC} sudo systemctl start ids-analytics-aggregator" +echo "" diff --git a/python_ml/analytics_aggregator.py b/python_ml/analytics_aggregator.py new file mode 100644 index 0000000..31edd70 --- /dev/null +++ b/python_ml/analytics_aggregator.py @@ -0,0 +1,318 @@ +""" +Network Analytics Aggregator +Aggrega statistiche traffico (normale + attacchi) ogni ora e giorno +Mantiene dati permanenti per analytics long-term +""" + +import psycopg2 +from psycopg2.extras import RealDictCursor +import os +from datetime import datetime, timedelta +import json +from typing import Dict, List, Optional +from collections import defaultdict +import sys + + +class AnalyticsAggregator: + """ + Aggregatore analytics per traffico normale + attacchi + Salva statistiche permanenti in network_analytics + """ + + def __init__(self): + self.db_params = { + 'host': os.getenv('PGHOST', 'localhost'), + 'port': int(os.getenv('PGPORT', 5432)), + 'database': os.getenv('PGDATABASE', 'ids_db'), + 'user': os.getenv('PGUSER', 'ids'), + 'password': os.getenv('PGPASSWORD', ''), + } + + def get_connection(self): + """Crea connessione database""" + return psycopg2.connect(**self.db_params) + + def aggregate_hourly(self, target_hour: Optional[datetime] = None): + """ + Aggrega statistiche per una specifica ora + Se target_hour รจ None, usa l'ora precedente + """ + if target_hour is None: + # Usa ora precedente (es: se ora sono le 10:30, aggrega le 09:00-10:00) + now = datetime.now() + target_hour = now.replace(minute=0, second=0, microsecond=0) - timedelta(hours=1) + + hour_start = target_hour + hour_end = hour_start + timedelta(hours=1) + + print(f"[ANALYTICS] Aggregazione oraria: {hour_start.strftime('%Y-%m-%d %H:00')}") + + conn = self.get_connection() + cursor = conn.cursor(cursor_factory=RealDictCursor) + + try: + # 1. Analizza network_logs nell'ora target + cursor.execute(""" + SELECT + source_ip, + COUNT(*) as packets, + SUM(packet_length) as bytes + FROM network_logs + WHERE timestamp >= %s AND timestamp < %s + GROUP BY source_ip + """, (hour_start, hour_end)) + + traffic_by_ip = {row['source_ip']: row for row in cursor.fetchall()} + + if not traffic_by_ip: + print(f"[ANALYTICS] Nessun traffico nell'ora {hour_start.strftime('%H:00')}") + cursor.close() + conn.close() + return + + # 2. Identifica IP attaccanti (detections nell'ora) + cursor.execute(""" + SELECT DISTINCT source_ip, anomaly_type, risk_score, country + FROM detections + WHERE detected_at >= %s AND detected_at < %s + """, (hour_start, hour_end)) + + attacker_ips = {} + attacks_by_type = defaultdict(int) + + for row in cursor.fetchall(): + ip = row['source_ip'] + attacker_ips[ip] = row + attacks_by_type[row['anomaly_type']] += 1 + + # 3. Classifica traffico: normale vs attacco + total_packets = 0 + total_bytes = 0 + normal_packets = 0 + normal_bytes = 0 + attack_packets = 0 + attack_bytes = 0 + + traffic_by_country = defaultdict(lambda: {'normal': 0, 'attacks': 0}) + attacks_by_country = defaultdict(int) + + top_normal = [] + top_attackers = [] + + for ip, stats in traffic_by_ip.items(): + packets = stats['packets'] + bytes_count = stats['bytes'] or 0 + + total_packets += packets + total_bytes += bytes_count + + if ip in attacker_ips: + # IP attaccante + attack_packets += packets + attack_bytes += bytes_count + + country = attacker_ips[ip].get('country') + if country: + traffic_by_country[country]['attacks'] += packets + attacks_by_country[country] += 1 + + top_attackers.append({ + 'ip': ip, + 'country': country, + 'risk_score': float(attacker_ips[ip]['risk_score']), + 'packets': packets, + 'bytes': bytes_count + }) + else: + # IP normale + normal_packets += packets + normal_bytes += bytes_count + + # Lookup paese per IP normale (da detections precedenti o geo cache) + cursor.execute(""" + SELECT country FROM detections + WHERE source_ip = %s AND country IS NOT NULL + ORDER BY detected_at DESC LIMIT 1 + """, (ip,)) + + geo_row = cursor.fetchone() + country = geo_row['country'] if geo_row else None + + if country: + traffic_by_country[country]['normal'] += packets + + top_normal.append({ + 'ip': ip, + 'packets': packets, + 'bytes': bytes_count, + 'country': country + }) + + # 4. Top 10 per categoria + top_normal = sorted(top_normal, key=lambda x: x['packets'], reverse=True)[:10] + top_attackers = sorted(top_attackers, key=lambda x: x['risk_score'], reverse=True)[:10] + + # 5. Salva aggregazione + cursor.execute(""" + INSERT INTO network_analytics ( + date, hour, + total_packets, total_bytes, unique_ips, + normal_packets, normal_bytes, normal_unique_ips, top_normal_ips, + attack_packets, attack_bytes, attack_unique_ips, + attacks_by_country, attacks_by_type, top_attackers, + traffic_by_country + ) + VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (date, hour) DO UPDATE SET + total_packets = EXCLUDED.total_packets, + total_bytes = EXCLUDED.total_bytes, + unique_ips = EXCLUDED.unique_ips, + normal_packets = EXCLUDED.normal_packets, + normal_bytes = EXCLUDED.normal_bytes, + normal_unique_ips = EXCLUDED.normal_unique_ips, + top_normal_ips = EXCLUDED.top_normal_ips, + attack_packets = EXCLUDED.attack_packets, + attack_bytes = EXCLUDED.attack_bytes, + attack_unique_ips = EXCLUDED.attack_unique_ips, + attacks_by_country = EXCLUDED.attacks_by_country, + attacks_by_type = EXCLUDED.attacks_by_type, + top_attackers = EXCLUDED.top_attackers, + traffic_by_country = EXCLUDED.traffic_by_country + """, ( + target_hour.date(), + target_hour.hour, + total_packets, + total_bytes, + len(traffic_by_ip), + normal_packets, + normal_bytes, + len(traffic_by_ip) - len(attacker_ips), + json.dumps(top_normal), + attack_packets, + attack_bytes, + len(attacker_ips), + json.dumps(dict(attacks_by_country)), + json.dumps(dict(attacks_by_type)), + json.dumps(top_attackers), + json.dumps({k: dict(v) for k, v in traffic_by_country.items()}) + )) + + conn.commit() + + print(f"[ANALYTICS] โœ… Aggregazione completata:") + print(f" - Totale: {total_packets} pacchetti, {len(traffic_by_ip)} IP unici") + print(f" - Normale: {normal_packets} pacchetti ({normal_packets*100//total_packets if total_packets else 0}%)") + print(f" - Attacchi: {attack_packets} pacchetti ({attack_packets*100//total_packets if total_packets else 0}%), {len(attacker_ips)} IP") + + except Exception as e: + print(f"[ANALYTICS] โŒ Errore aggregazione oraria: {e}") + conn.rollback() + raise + finally: + cursor.close() + conn.close() + + def aggregate_daily(self, target_date: Optional[datetime] = None): + """ + Aggrega statistiche giornaliere (somma delle ore) + Se target_date รจ None, usa giorno precedente + """ + if target_date is None: + target_date = datetime.now().date() - timedelta(days=1) + + print(f"[ANALYTICS] Aggregazione giornaliera: {target_date}") + + conn = self.get_connection() + cursor = conn.cursor(cursor_factory=RealDictCursor) + + try: + # Somma aggregazioni orarie del giorno + cursor.execute(""" + SELECT + SUM(total_packets) as total_packets, + SUM(total_bytes) as total_bytes, + MAX(unique_ips) as unique_ips, + SUM(normal_packets) as normal_packets, + SUM(normal_bytes) as normal_bytes, + MAX(normal_unique_ips) as normal_unique_ips, + SUM(attack_packets) as attack_packets, + SUM(attack_bytes) as attack_bytes, + MAX(attack_unique_ips) as attack_unique_ips + FROM network_analytics + WHERE date = %s AND hour IS NOT NULL + """, (target_date,)) + + daily_stats = cursor.fetchone() + + if not daily_stats or not daily_stats['total_packets']: + print(f"[ANALYTICS] Nessun dato per {target_date}") + cursor.close() + conn.close() + return + + # Merge JSON fields (countries, types, top IPs) + # TODO: Implementare merge intelligente se necessario + + # Salva aggregazione giornaliera (hour = NULL) + cursor.execute(""" + INSERT INTO network_analytics ( + date, hour, + total_packets, total_bytes, unique_ips, + normal_packets, normal_bytes, normal_unique_ips, + attack_packets, attack_bytes, attack_unique_ips + ) + VALUES (%s, NULL, %s, %s, %s, %s, %s, %s, %s, %s, %s) + ON CONFLICT (date, hour) DO UPDATE SET + total_packets = EXCLUDED.total_packets, + total_bytes = EXCLUDED.total_bytes, + unique_ips = EXCLUDED.unique_ips, + normal_packets = EXCLUDED.normal_packets, + normal_bytes = EXCLUDED.normal_bytes, + normal_unique_ips = EXCLUDED.normal_unique_ips, + attack_packets = EXCLUDED.attack_packets, + attack_bytes = EXCLUDED.attack_bytes, + attack_unique_ips = EXCLUDED.attack_unique_ips + """, ( + target_date, + daily_stats['total_packets'], + daily_stats['total_bytes'], + daily_stats['unique_ips'], + daily_stats['normal_packets'], + daily_stats['normal_bytes'], + daily_stats['normal_unique_ips'], + daily_stats['attack_packets'], + daily_stats['attack_bytes'], + daily_stats['attack_unique_ips'] + )) + + conn.commit() + print(f"[ANALYTICS] โœ… Aggregazione giornaliera completata") + + except Exception as e: + print(f"[ANALYTICS] โŒ Errore aggregazione giornaliera: {e}") + conn.rollback() + raise + finally: + cursor.close() + conn.close() + + +def main(): + """Entry point""" + aggregator = AnalyticsAggregator() + + if len(sys.argv) > 1: + if sys.argv[1] == 'hourly': + aggregator.aggregate_hourly() + elif sys.argv[1] == 'daily': + aggregator.aggregate_daily() + else: + print("Usage: python analytics_aggregator.py [hourly|daily]") + else: + # Default: aggregazione oraria + aggregator.aggregate_hourly() + + +if __name__ == '__main__': + main() diff --git a/replit.md b/replit.md index ab99937..7fa41e1 100644 --- a/replit.md +++ b/replit.md @@ -45,4 +45,26 @@ The IDS employs a React-based frontend for real-time monitoring, detection visua - **Neon Database**: Cloud-native PostgreSQL service (used in Replit). - **pg (Node.js driver)**: Standard PostgreSQL driver for Node.js (used in AlmaLinux). - **psycopg2**: PostgreSQL adapter for Python. -- **ip-api.com**: External API for IP geolocation data. \ No newline at end of file +- **ip-api.com**: External API for IP geolocation data. +- **Recharts**: Charting library for analytics visualization. + +## Recent Updates (Novembre 2025) + +### ๐Ÿ“Š Network Analytics & Dashboard System (22 Nov 2025 - 15:00) +- **Feature Completa**: Sistema analytics con traffico normale + attacchi, visualizzazioni grafiche avanzate, dati permanenti +- **Componenti**: + 1. **Database**: `network_analytics` table con aggregazioni orarie/giornaliere permanenti + 2. **Aggregatore Python**: `analytics_aggregator.py` classifica traffico ogni ora + 3. **Systemd Timer**: Esecuzione automatica ogni ora (:05 minuti) + 4. **API**: `/api/analytics/recent` e `/api/analytics/range` + 5. **Frontend**: Dashboard Live (real-time 3 giorni) + Analytics Storici (permanente) +- **Grafici**: Area Chart, Pie Chart, Bar Chart, Line Chart, Real-time Stream +- **Flag Emoji**: ๐Ÿ‡ฎ๐Ÿ‡น๐Ÿ‡บ๐Ÿ‡ธ๐Ÿ‡ท๐Ÿ‡บ๐Ÿ‡จ๐Ÿ‡ณ per identificazione immediata paese origine +- **Deploy**: Migration 005 + `./deployment/setup_analytics_timer.sh` + +### ๐ŸŒ IP Geolocation Integration (22 Nov 2025 - 13:00) +- **Feature**: Informazioni geografiche complete (paese, cittร , organizzazione, AS) per ogni IP +- **API**: ip-api.com con batch async lookup (100 IP in ~1.5s invece di 150s!) +- **Performance**: Caching intelligente + fallback robusto +- **Display**: Globe/Building/MapPin icons nella pagina Detections +- **Deploy**: Migration 004 + restart ML backend \ No newline at end of file diff --git a/server/routes.ts b/server/routes.ts index 2789039..608b86e 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -133,6 +133,37 @@ export async function registerRoutes(app: Express): Promise { } }); + // Network Analytics + app.get("/api/analytics/recent", async (req, res) => { + try { + const days = parseInt(req.query.days as string) || 3; + const hourly = req.query.hourly === 'true'; + const analytics = await storage.getRecentAnalytics(days, hourly); + res.json(analytics); + } catch (error) { + console.error('[DB ERROR] Failed to fetch recent analytics:', error); + res.status(500).json({ error: "Failed to fetch analytics" }); + } + }); + + app.get("/api/analytics/range", async (req, res) => { + try { + const startDate = new Date(req.query.start as string); + const endDate = new Date(req.query.end as string); + const hourly = req.query.hourly === 'true'; + + if (isNaN(startDate.getTime()) || isNaN(endDate.getTime())) { + return res.status(400).json({ error: "Invalid date range" }); + } + + const analytics = await storage.getAnalyticsByDateRange(startDate, endDate, hourly); + res.json(analytics); + } catch (error) { + console.error('[DB ERROR] Failed to fetch analytics range:', error); + res.status(500).json({ error: "Failed to fetch analytics" }); + } + }); + // Stats app.get("/api/stats", async (req, res) => { try { diff --git a/server/storage.ts b/server/storage.ts index 9acc087..cfe2321 100644 --- a/server/storage.ts +++ b/server/storage.ts @@ -4,6 +4,7 @@ import { detections, whitelist, trainingHistory, + networkAnalytics, type Router, type InsertRouter, type NetworkLog, @@ -14,6 +15,7 @@ import { type InsertWhitelist, type TrainingHistory, type InsertTrainingHistory, + type NetworkAnalytics, } from "@shared/schema"; import { db } from "./db"; import { eq, desc, and, gte, sql, inArray } from "drizzle-orm"; @@ -51,6 +53,10 @@ export interface IStorage { createTrainingHistory(history: InsertTrainingHistory): Promise; getLatestTraining(): Promise; + // Network Analytics + getAnalyticsByDateRange(startDate: Date, endDate: Date, hourly?: boolean): Promise; + getRecentAnalytics(days: number, hourly?: boolean): Promise; + // System testConnection(): Promise; } @@ -226,7 +232,32 @@ export class DatabaseStorage implements IStorage { return history || undefined; } - // System + // Network Analytics + async getAnalyticsByDateRange(startDate: Date, endDate: Date, hourly: boolean = false): Promise { + const hourCondition = hourly + ? sql`hour IS NOT NULL` + : sql`hour IS NULL`; + + return await db + .select() + .from(networkAnalytics) + .where( + and( + gte(networkAnalytics.date, startDate), + sql`${networkAnalytics.date} <= ${endDate}`, + hourCondition + ) + ) + .orderBy(desc(networkAnalytics.date), desc(networkAnalytics.hour)); + } + + async getRecentAnalytics(days: number, hourly: boolean = false): Promise { + const startDate = new Date(); + startDate.setDate(startDate.getDate() - days); + + return this.getAnalyticsByDateRange(startDate, new Date(), hourly); + } + async testConnection(): Promise { try { await db.execute(sql`SELECT 1`); diff --git a/shared/schema.ts b/shared/schema.ts index bd4dace..7fbdc43 100644 --- a/shared/schema.ts +++ b/shared/schema.ts @@ -1,5 +1,5 @@ import { sql, relations } from "drizzle-orm"; -import { pgTable, text, varchar, integer, timestamp, decimal, boolean, index } from "drizzle-orm/pg-core"; +import { pgTable, text, varchar, integer, timestamp, decimal, boolean, index, bigint } from "drizzle-orm/pg-core"; import { createInsertSchema } from "drizzle-zod"; import { z } from "zod"; @@ -89,6 +89,40 @@ export const trainingHistory = pgTable("training_history", { trainedAt: timestamp("trained_at").defaultNow().notNull(), }); +// Network analytics - aggregazioni permanenti per statistiche long-term +export const networkAnalytics = pgTable("network_analytics", { + id: varchar("id").primaryKey().default(sql`gen_random_uuid()`), + date: timestamp("date", { mode: 'date' }).notNull(), + hour: integer("hour"), // NULL = giornaliera, 0-23 = oraria + + // Traffico totale + totalPackets: integer("total_packets").notNull().default(0), + totalBytes: bigint("total_bytes", { mode: 'number' }).notNull().default(0), + uniqueIps: integer("unique_ips").notNull().default(0), + + // Traffico normale (non anomalo) + normalPackets: integer("normal_packets").notNull().default(0), + normalBytes: bigint("normal_bytes", { mode: 'number' }).notNull().default(0), + normalUniqueIps: integer("normal_unique_ips").notNull().default(0), + topNormalIps: text("top_normal_ips"), // JSON: [{ip, packets, bytes, country}] + + // Attacchi/Anomalie + attackPackets: integer("attack_packets").notNull().default(0), + attackBytes: bigint("attack_bytes", { mode: 'number' }).notNull().default(0), + attackUniqueIps: integer("attack_unique_ips").notNull().default(0), + attacksByCountry: text("attacks_by_country"), // JSON: {IT: 5, RU: 30} + attacksByType: text("attacks_by_type"), // JSON: {ddos: 10, port_scan: 5} + topAttackers: text("top_attackers"), // JSON: [{ip, country, risk_score, packets}] + + // Dettagli geografici (tutto il traffico) + trafficByCountry: text("traffic_by_country"), // JSON: {IT: {normal: 100, attacks: 5}} + + createdAt: timestamp("created_at").defaultNow().notNull(), +}, (table) => ({ + dateHourIdx: index("network_analytics_date_hour_idx").on(table.date, table.hour), + dateIdx: index("network_analytics_date_idx").on(table.date), +})); + // Schema version tracking for database migrations export const schemaVersion = pgTable("schema_version", { id: integer("id").primaryKey().default(1), @@ -135,6 +169,11 @@ export const insertSchemaVersionSchema = createInsertSchema(schemaVersion).omit( appliedAt: true, }); +export const insertNetworkAnalyticsSchema = createInsertSchema(networkAnalytics).omit({ + id: true, + createdAt: true, +}); + // Types export type Router = typeof routers.$inferSelect; export type InsertRouter = z.infer; @@ -153,3 +192,6 @@ export type InsertTrainingHistory = z.infer; export type SchemaVersion = typeof schemaVersion.$inferSelect; export type InsertSchemaVersion = z.infer; + +export type NetworkAnalytics = typeof networkAnalytics.$inferSelect; +export type InsertNetworkAnalytics = z.infer;