diff --git a/client/src/App.tsx b/client/src/App.tsx index 4b4cdb5..85e1d61 100644 --- a/client/src/App.tsx +++ b/client/src/App.tsx @@ -4,13 +4,14 @@ import { QueryClientProvider } from "@tanstack/react-query"; import { Toaster } from "@/components/ui/toaster"; import { TooltipProvider } from "@/components/ui/tooltip"; import { SidebarProvider, Sidebar, SidebarContent, SidebarGroup, SidebarGroupContent, SidebarGroupLabel, SidebarMenu, SidebarMenuButton, SidebarMenuItem, SidebarTrigger } from "@/components/ui/sidebar"; -import { LayoutDashboard, AlertTriangle, Server, Shield, Brain, Menu, Activity, BarChart3, TrendingUp } from "lucide-react"; +import { LayoutDashboard, AlertTriangle, Server, Shield, Brain, Menu, Activity, BarChart3, TrendingUp, List } from "lucide-react"; import Dashboard from "@/pages/Dashboard"; import Detections from "@/pages/Detections"; import DashboardLive from "@/pages/DashboardLive"; import AnalyticsHistory from "@/pages/AnalyticsHistory"; import Routers from "@/pages/Routers"; import Whitelist from "@/pages/Whitelist"; +import PublicLists from "@/pages/PublicLists"; import Training from "@/pages/Training"; import Services from "@/pages/Services"; import NotFound from "@/pages/not-found"; @@ -23,6 +24,7 @@ const menuItems = [ { title: "Training ML", url: "/training", icon: Brain }, { title: "Router", url: "/routers", icon: Server }, { title: "Whitelist", url: "/whitelist", icon: Shield }, + { title: "Liste Pubbliche", url: "/public-lists", icon: List }, { title: "Servizi", url: "/services", icon: TrendingUp }, ]; @@ -62,6 +64,7 @@ function Router() { + diff --git a/client/src/pages/PublicLists.tsx b/client/src/pages/PublicLists.tsx new file mode 100644 index 0000000..d42704d --- /dev/null +++ b/client/src/pages/PublicLists.tsx @@ -0,0 +1,382 @@ +import { useQuery, useMutation } from "@tanstack/react-query"; +import { Card, CardContent, CardDescription, CardHeader, CardTitle } from "@/components/ui/card"; +import { Button } from "@/components/ui/button"; +import { Badge } from "@/components/ui/badge"; +import { Table, TableBody, TableCell, TableHead, TableHeader, TableRow } from "@/components/ui/table"; +import { Dialog, DialogContent, DialogDescription, DialogHeader, DialogTitle, DialogTrigger } from "@/components/ui/dialog"; +import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form"; +import { Input } from "@/components/ui/input"; +import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select"; +import { Switch } from "@/components/ui/switch"; +import { useForm } from "react-hook-form"; +import { zodResolver } from "@hookform/resolvers/zod"; +import { z } from "zod"; +import { RefreshCw, Plus, Trash2, Edit, CheckCircle2, XCircle, AlertTriangle, Clock } from "lucide-react"; +import { apiRequest, queryClient } from "@/lib/queryClient"; +import { useToast } from "@/hooks/use-toast"; +import { formatDistanceToNow } from "date-fns"; +import { it } from "date-fns/locale"; +import { useState } from "react"; + +const listFormSchema = z.object({ + name: z.string().min(1, "Nome richiesto"), + type: z.enum(["blacklist", "whitelist"], { + required_error: "Tipo richiesto", + }), + url: z.string().url("URL non valida"), + enabled: z.boolean().default(true), + fetchIntervalMinutes: z.number().min(1).max(1440).default(10), +}); + +type ListFormValues = z.infer; + +export default function PublicLists() { + const { toast } = useToast(); + const [isAddDialogOpen, setIsAddDialogOpen] = useState(false); + const [editingList, setEditingList] = useState(null); + + const { data: lists, isLoading } = useQuery({ + queryKey: ["/api/public-lists"], + }); + + const form = useForm({ + resolver: zodResolver(listFormSchema), + defaultValues: { + name: "", + type: "blacklist", + url: "", + enabled: true, + fetchIntervalMinutes: 10, + }, + }); + + const createMutation = useMutation({ + mutationFn: (data: ListFormValues) => + apiRequest("/api/public-lists", { + method: "POST", + body: JSON.stringify(data), + }), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["/api/public-lists"] }); + toast({ + title: "Lista creata", + description: "La lista è stata aggiunta con successo", + }); + setIsAddDialogOpen(false); + form.reset(); + }, + onError: (error: any) => { + toast({ + title: "Errore", + description: error.message || "Impossibile creare la lista", + variant: "destructive", + }); + }, + }); + + const updateMutation = useMutation({ + mutationFn: ({ id, data }: { id: string; data: Partial }) => + apiRequest(`/api/public-lists/${id}`, { + method: "PATCH", + body: JSON.stringify(data), + }), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["/api/public-lists"] }); + toast({ + title: "Lista aggiornata", + description: "Le modifiche sono state salvate", + }); + setEditingList(null); + }, + }); + + const deleteMutation = useMutation({ + mutationFn: (id: string) => + apiRequest(`/api/public-lists/${id}`, { + method: "DELETE", + }), + onSuccess: () => { + queryClient.invalidateQueries({ queryKey: ["/api/public-lists"] }); + toast({ + title: "Lista eliminata", + description: "La lista è stata rimossa", + }); + }, + onError: (error: any) => { + toast({ + title: "Errore", + description: error.message || "Impossibile eliminare la lista", + variant: "destructive", + }); + }, + }); + + const syncMutation = useMutation({ + mutationFn: (id: string) => + apiRequest(`/api/public-lists/${id}/sync`, { + method: "POST", + }), + onSuccess: () => { + toast({ + title: "Sync avviato", + description: "La sincronizzazione manuale è stata richiesta", + }); + }, + }); + + const toggleEnabled = (id: string, enabled: boolean) => { + updateMutation.mutate({ id, data: { enabled } }); + }; + + const onSubmit = (data: ListFormValues) => { + createMutation.mutate(data); + }; + + const getStatusBadge = (list: any) => { + if (!list.enabled) { + return Disabilitata; + } + + if (list.errorCount > 5) { + return Errori; + } + + if (list.lastSuccess) { + return OK; + } + + return In attesa; + }; + + const getTypeBadge = (type: string) => { + if (type === "blacklist") { + return Blacklist; + } + return Whitelist; + }; + + if (isLoading) { + return ( +
+ + + Caricamento... + + +
+ ); + } + + return ( +
+
+
+

Liste Pubbliche

+

+ Gestione sorgenti blacklist e whitelist esterne (aggiornamento ogni 10 minuti) +

+
+ + + + + + + Aggiungi Lista Pubblica + + Configura una nuova sorgente blacklist o whitelist + + +
+ + ( + + Nome + + + + + + )} + /> + ( + + Tipo + + + + )} + /> + ( + + URL + + + + + + )} + /> + ( + + Intervallo Sync (minuti) + + field.onChange(parseInt(e.target.value))} + data-testid="input-list-interval" + /> + + + + )} + /> + ( + + Abilitata + + + + + )} + /> +
+ + +
+ + +
+ +
+ + + + Sorgenti Configurate + + {lists?.length || 0} liste configurate + + + + + + + Nome + Tipo + Stato + IP Totali + IP Attivi + Ultimo Sync + Azioni + + + + {lists?.map((list: any) => ( + + +
+
{list.name}
+
+ {list.url} +
+
+ + {getTypeBadge(list.type)} + {getStatusBadge(list)} + {list.totalIps?.toLocaleString() || 0} + {list.activeIps?.toLocaleString() || 0} + + {list.lastSuccess ? ( + + {formatDistanceToNow(new Date(list.lastSuccess), { + addSuffix: true, + locale: it, + })} + + ) : ( + Mai + )} + + +
+ toggleEnabled(list.id, checked)} + data-testid={`switch-enable-${list.id}`} + /> + + +
+ + + ))} + {(!lists || lists.length === 0) && ( + + + Nessuna lista configurata. Aggiungi la prima lista. + + + )} + +
+ + +
+ ); +} diff --git a/deployment/migrations/006_add_public_lists.sql b/deployment/migrations/006_add_public_lists.sql new file mode 100644 index 0000000..4ef2f20 --- /dev/null +++ b/deployment/migrations/006_add_public_lists.sql @@ -0,0 +1,146 @@ +-- Migration 006: Add Public Lists Integration +-- Description: Adds blacklist/whitelist public sources with auto-sync support +-- Author: IDS System +-- Date: 2024-11-26 + +BEGIN; + +-- ============================================================================ +-- 1. CREATE NEW TABLES +-- ============================================================================ + +-- Public threat/whitelist sources configuration +CREATE TABLE IF NOT EXISTS public_lists ( + id VARCHAR PRIMARY KEY DEFAULT gen_random_uuid(), + name TEXT NOT NULL, + type TEXT NOT NULL CHECK (type IN ('blacklist', 'whitelist')), + url TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT true, + fetch_interval_minutes INTEGER NOT NULL DEFAULT 10, + last_fetch TIMESTAMP, + last_success TIMESTAMP, + total_ips INTEGER NOT NULL DEFAULT 0, + active_ips INTEGER NOT NULL DEFAULT 0, + error_count INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + created_at TIMESTAMP NOT NULL DEFAULT NOW() +); + +CREATE INDEX public_lists_type_idx ON public_lists(type); +CREATE INDEX public_lists_enabled_idx ON public_lists(enabled); + +-- Public blacklist IPs from external sources +CREATE TABLE IF NOT EXISTS public_blacklist_ips ( + id VARCHAR PRIMARY KEY DEFAULT gen_random_uuid(), + ip_address TEXT NOT NULL, + cidr_range TEXT, + list_id VARCHAR NOT NULL REFERENCES public_lists(id) ON DELETE CASCADE, + first_seen TIMESTAMP NOT NULL DEFAULT NOW(), + last_seen TIMESTAMP NOT NULL DEFAULT NOW(), + is_active BOOLEAN NOT NULL DEFAULT true +); + +CREATE INDEX public_blacklist_ip_idx ON public_blacklist_ips(ip_address); +CREATE INDEX public_blacklist_list_idx ON public_blacklist_ips(list_id); +CREATE INDEX public_blacklist_active_idx ON public_blacklist_ips(is_active); +CREATE UNIQUE INDEX public_blacklist_ip_list_key ON public_blacklist_ips(ip_address, list_id); + +-- ============================================================================ +-- 2. ALTER EXISTING TABLES +-- ============================================================================ + +-- Extend detections table with public list source tracking +ALTER TABLE detections +ADD COLUMN IF NOT EXISTS detection_source TEXT NOT NULL DEFAULT 'ml_model', +ADD COLUMN IF NOT EXISTS blacklist_id VARCHAR; + +CREATE INDEX IF NOT EXISTS detection_source_idx ON detections(detection_source); + +-- Add check constraint for valid detection sources +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'detections_source_check' + ) THEN + ALTER TABLE detections + ADD CONSTRAINT detections_source_check + CHECK (detection_source IN ('ml_model', 'public_blacklist', 'hybrid')); + END IF; +END $$; + +-- Extend whitelist table with source tracking +ALTER TABLE whitelist +ADD COLUMN IF NOT EXISTS source TEXT NOT NULL DEFAULT 'manual', +ADD COLUMN IF NOT EXISTS list_id VARCHAR; + +CREATE INDEX IF NOT EXISTS whitelist_source_idx ON whitelist(source); + +-- Add check constraint for valid whitelist sources +DO $$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_constraint + WHERE conname = 'whitelist_source_check' + ) THEN + ALTER TABLE whitelist + ADD CONSTRAINT whitelist_source_check + CHECK (source IN ('manual', 'aws', 'gcp', 'cloudflare', 'iana', 'ntp', 'other')); + END IF; +END $$; + +-- ============================================================================ +-- 3. UPDATE SCHEMA VERSION +-- ============================================================================ + +-- Insert new version record +INSERT INTO schema_version (id, version, description) +VALUES (1, 6, 'Add public lists integration (blacklist/whitelist sources)') +ON CONFLICT (id) DO UPDATE +SET version = 6, + description = 'Add public lists integration (blacklist/whitelist sources)', + applied_at = NOW(); + +-- ============================================================================ +-- 4. SEED DEFAULT PUBLIC LISTS (OPTIONAL - commented for manual execution) +-- ============================================================================ + +/* +-- Blacklist sources +INSERT INTO public_lists (name, type, url, enabled, fetch_interval_minutes) VALUES +('Spamhaus DROP', 'blacklist', 'https://www.spamhaus.org/drop/drop.txt', true, 10), +('Talos Intelligence', 'blacklist', 'https://talosintelligence.com/documents/ip-blacklist', false, 10); + +-- Whitelist sources +INSERT INTO public_lists (name, type, url, enabled, fetch_interval_minutes) VALUES +('AWS IP Ranges', 'whitelist', 'https://ip-ranges.amazonaws.com/ip-ranges.json', true, 10), +('Google Cloud IP Ranges', 'whitelist', 'https://www.gstatic.com/ipranges/cloud.json', true, 10), +('Cloudflare IPs', 'whitelist', 'https://www.cloudflare.com/ips-v4', true, 10), +('IANA Root Servers', 'whitelist', 'https://www.iana.org/domains/root/servers', true, 10), +('NTP Pool', 'whitelist', 'https://www.ntppool.org/zone/@', false, 10); +*/ + +COMMIT; + +-- ============================================================================ +-- ROLLBACK PROCEDURE (if needed) +-- ============================================================================ + +/* +BEGIN; + +-- Remove new columns from existing tables +ALTER TABLE detections DROP COLUMN IF EXISTS detection_source; +ALTER TABLE detections DROP COLUMN IF EXISTS blacklist_id; +ALTER TABLE whitelist DROP COLUMN IF EXISTS source; +ALTER TABLE whitelist DROP COLUMN IF EXISTS list_id; + +-- Drop new tables (CASCADE removes FK constraints) +DROP TABLE IF EXISTS public_blacklist_ips CASCADE; +DROP TABLE IF EXISTS public_lists CASCADE; + +-- Revert schema version +UPDATE schema_version SET version = 5, description = 'Rollback from public lists integration' WHERE id = 1; + +COMMIT; +*/ diff --git a/deployment/systemd/ids-list-fetcher.service b/deployment/systemd/ids-list-fetcher.service new file mode 100644 index 0000000..a49c62a --- /dev/null +++ b/deployment/systemd/ids-list-fetcher.service @@ -0,0 +1,29 @@ +[Unit] +Description=IDS Public Lists Fetcher Service +Documentation=https://github.com/yourorg/ids +After=network.target postgresql.service + +[Service] +Type=oneshot +User=root +WorkingDirectory=/opt/ids/python_ml +Environment="PYTHONUNBUFFERED=1" +EnvironmentFile=/opt/ids/.env + +# Run list fetcher with virtual environment +ExecStart=/opt/ids/python_ml/venv/bin/python3 /opt/ids/python_ml/list_fetcher/run_fetcher.py + +# Logging +StandardOutput=journal +StandardError=journal +SyslogIdentifier=ids-list-fetcher + +# Security settings +PrivateTmp=true +NoNewPrivileges=true + +# Restart policy +Restart=no + +[Install] +WantedBy=multi-user.target diff --git a/deployment/systemd/ids-list-fetcher.timer b/deployment/systemd/ids-list-fetcher.timer new file mode 100644 index 0000000..a9cdf1e --- /dev/null +++ b/deployment/systemd/ids-list-fetcher.timer @@ -0,0 +1,13 @@ +[Unit] +Description=IDS Public Lists Fetcher Timer (every 10 minutes) +Documentation=https://github.com/yourorg/ids + +[Timer] +# Run every 10 minutes +OnCalendar=*:0/10 +OnBootSec=2min +AccuracySec=1min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/main.py b/main.py new file mode 100644 index 0000000..d7b9ef1 --- /dev/null +++ b/main.py @@ -0,0 +1,6 @@ +def main(): + print("Hello from repl-nix-workspace!") + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..49f0bca --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[project] +name = "repl-nix-workspace" +version = "0.1.0" +description = "Add your description here" +requires-python = ">=3.11" +dependencies = [ + "httpx>=0.28.1", +] diff --git a/python_ml/list_fetcher/__init__.py b/python_ml/list_fetcher/__init__.py new file mode 100644 index 0000000..a66e8bd --- /dev/null +++ b/python_ml/list_fetcher/__init__.py @@ -0,0 +1,2 @@ +# Public Lists Fetcher Module +# Handles download, parsing, and sync of public blacklist/whitelist sources diff --git a/python_ml/list_fetcher/fetcher.py b/python_ml/list_fetcher/fetcher.py new file mode 100644 index 0000000..b2b73f8 --- /dev/null +++ b/python_ml/list_fetcher/fetcher.py @@ -0,0 +1,391 @@ +import asyncio +import httpx +from datetime import datetime +from typing import Dict, List, Set, Tuple, Optional +import psycopg2 +from psycopg2.extras import execute_values +import os +import sys + +# Add parent directory to path for imports +sys.path.append(os.path.dirname(os.path.dirname(__file__))) + +from list_fetcher.parsers import parse_list + + +class ListFetcher: + """Fetches and synchronizes public IP lists""" + + def __init__(self, database_url: str): + self.database_url = database_url + self.timeout = 30.0 + self.max_retries = 3 + + def get_db_connection(self): + """Create database connection""" + return psycopg2.connect(self.database_url) + + async def fetch_url(self, url: str) -> Optional[str]: + """Download content from URL with retry logic""" + async with httpx.AsyncClient(timeout=self.timeout, follow_redirects=True) as client: + for attempt in range(self.max_retries): + try: + response = await client.get(url) + response.raise_for_status() + return response.text + except httpx.HTTPError as e: + if attempt == self.max_retries - 1: + raise Exception(f"HTTP error after {self.max_retries} attempts: {e}") + await asyncio.sleep(2 ** attempt) # Exponential backoff + except Exception as e: + if attempt == self.max_retries - 1: + raise Exception(f"Download failed: {e}") + await asyncio.sleep(2 ** attempt) + return None + + def get_enabled_lists(self) -> List[Dict]: + """Get all enabled public lists from database""" + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT id, name, type, url, fetch_interval_minutes + FROM public_lists + WHERE enabled = true + ORDER BY type, name + """) + if cur.description is None: + return [] + columns = [desc[0] for desc in cur.description] + return [dict(zip(columns, row)) for row in cur.fetchall()] + finally: + conn.close() + + def get_existing_ips(self, list_id: str, list_type: str) -> Set[str]: + """Get existing IPs for a list from database""" + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + if list_type == 'blacklist': + cur.execute(""" + SELECT ip_address + FROM public_blacklist_ips + WHERE list_id = %s AND is_active = true + """, (list_id,)) + else: # whitelist + cur.execute(""" + SELECT ip_address + FROM whitelist + WHERE list_id = %s AND active = true + """, (list_id,)) + + return {row[0] for row in cur.fetchall()} + finally: + conn.close() + + def sync_blacklist_ips(self, list_id: str, new_ips: Set[Tuple[str, Optional[str]]]): + """Sync blacklist IPs: add new, mark inactive old ones""" + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + # Get existing IPs + existing = self.get_existing_ips(list_id, 'blacklist') + new_ip_addresses = {ip for ip, _ in new_ips} + + # Calculate diff + to_add = new_ip_addresses - existing + to_deactivate = existing - new_ip_addresses + to_update = existing & new_ip_addresses + + # Mark old IPs as inactive + if to_deactivate: + cur.execute(""" + UPDATE public_blacklist_ips + SET is_active = false + WHERE list_id = %s AND ip_address = ANY(%s) + """, (list_id, list(to_deactivate))) + + # Update last_seen for existing active IPs + if to_update: + cur.execute(""" + UPDATE public_blacklist_ips + SET last_seen = NOW() + WHERE list_id = %s AND ip_address = ANY(%s) + """, (list_id, list(to_update))) + + # Add new IPs + if to_add: + values = [] + for ip, cidr in new_ips: + if ip in to_add: + values.append((ip, cidr, list_id)) + + execute_values(cur, """ + INSERT INTO public_blacklist_ips (ip_address, cidr_range, list_id) + VALUES %s + ON CONFLICT (ip_address, list_id) DO UPDATE + SET is_active = true, last_seen = NOW() + """, values) + + # Update list stats + cur.execute(""" + UPDATE public_lists + SET total_ips = %s, + active_ips = %s, + last_success = NOW() + WHERE id = %s + """, (len(new_ip_addresses), len(new_ip_addresses), list_id)) + + conn.commit() + return len(to_add), len(to_deactivate), len(to_update) + + except Exception as e: + conn.rollback() + raise e + finally: + conn.close() + + def sync_whitelist_ips(self, list_id: str, list_name: str, new_ips: Set[Tuple[str, Optional[str]]]): + """Sync whitelist IPs: add new, deactivate old ones""" + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + # Get existing IPs + existing = self.get_existing_ips(list_id, 'whitelist') + new_ip_addresses = {ip for ip, _ in new_ips} + + # Calculate diff + to_add = new_ip_addresses - existing + to_deactivate = existing - new_ip_addresses + to_update = existing & new_ip_addresses + + # Determine source name from list name + source = 'other' + list_lower = list_name.lower() + if 'aws' in list_lower: + source = 'aws' + elif 'gcp' in list_lower or 'google' in list_lower: + source = 'gcp' + elif 'cloudflare' in list_lower: + source = 'cloudflare' + elif 'iana' in list_lower: + source = 'iana' + elif 'ntp' in list_lower: + source = 'ntp' + + # Mark old IPs as inactive + if to_deactivate: + cur.execute(""" + UPDATE whitelist + SET active = false + WHERE list_id = %s AND ip_address = ANY(%s) + """, (list_id, list(to_deactivate))) + + # Add new IPs + if to_add: + values = [] + for ip, cidr in new_ips: + if ip in to_add: + comment = f"Auto-imported from {list_name}" + if cidr: + comment += f" (CIDR: {cidr})" + values.append((ip, comment, source, list_id)) + + execute_values(cur, """ + INSERT INTO whitelist (ip_address, comment, source, list_id) + VALUES %s + ON CONFLICT (ip_address) DO UPDATE + SET active = true, source = EXCLUDED.source, list_id = EXCLUDED.list_id + """, values) + + # Update list stats + cur.execute(""" + UPDATE public_lists + SET total_ips = %s, + active_ips = %s, + last_success = NOW() + WHERE id = %s + """, (len(new_ip_addresses), len(new_ip_addresses), list_id)) + + conn.commit() + return len(to_add), len(to_deactivate), len(to_update) + + except Exception as e: + conn.rollback() + raise e + finally: + conn.close() + + async def fetch_and_sync_list(self, list_config: Dict) -> Dict: + """Fetch and sync a single list""" + list_id = list_config['id'] + list_name = list_config['name'] + list_type = list_config['type'] + url = list_config['url'] + + result = { + 'list_id': list_id, + 'list_name': list_name, + 'success': False, + 'added': 0, + 'removed': 0, + 'updated': 0, + 'error': None + } + + conn = self.get_db_connection() + + try: + # Update last_fetch timestamp + with conn.cursor() as cur: + cur.execute(""" + UPDATE public_lists + SET last_fetch = NOW() + WHERE id = %s + """, (list_id,)) + conn.commit() + + # Download content + print(f"[{datetime.now().strftime('%H:%M:%S')}] Downloading {list_name} from {url}...") + content = await self.fetch_url(url) + + if not content: + raise Exception("Empty response from server") + + # Parse IPs + print(f"[{datetime.now().strftime('%H:%M:%S')}] Parsing {list_name}...") + ips = parse_list(list_name, content) + + if not ips: + raise Exception("No valid IPs found in list") + + print(f"[{datetime.now().strftime('%H:%M:%S')}] Found {len(ips)} IPs, syncing to database...") + + # Sync to database + if list_type == 'blacklist': + added, removed, updated = self.sync_blacklist_ips(list_id, ips) + else: + added, removed, updated = self.sync_whitelist_ips(list_id, list_name, ips) + + result.update({ + 'success': True, + 'added': added, + 'removed': removed, + 'updated': updated + }) + + print(f"[{datetime.now().strftime('%H:%M:%S')}] ✓ {list_name}: +{added} -{removed} ~{updated}") + + # Reset error count on success + with conn.cursor() as cur: + cur.execute(""" + UPDATE public_lists + SET error_count = 0, last_error = NULL + WHERE id = %s + """, (list_id,)) + conn.commit() + + except Exception as e: + error_msg = str(e) + result['error'] = error_msg + print(f"[{datetime.now().strftime('%H:%M:%S')}] ✗ {list_name}: {error_msg}") + + # Increment error count + with conn.cursor() as cur: + cur.execute(""" + UPDATE public_lists + SET error_count = error_count + 1, + last_error = %s + WHERE id = %s + """, (error_msg[:500], list_id)) + conn.commit() + + finally: + conn.close() + + return result + + async def fetch_all_lists(self) -> List[Dict]: + """Fetch and sync all enabled lists""" + print(f"\n{'='*60}") + print(f"[{datetime.now().strftime('%Y-%m-%d %H:%M:%S')}] PUBLIC LISTS SYNC") + print(f"{'='*60}\n") + + # Get enabled lists + lists = self.get_enabled_lists() + + if not lists: + print("No enabled lists found") + return [] + + print(f"Found {len(lists)} enabled lists\n") + + # Fetch all lists in parallel + tasks = [self.fetch_and_sync_list(list_config) for list_config in lists] + results = await asyncio.gather(*tasks, return_exceptions=True) + + # Summary + print(f"\n{'='*60}") + print("SYNC SUMMARY") + print(f"{'='*60}") + + success_count = sum(1 for r in results if isinstance(r, dict) and r.get('success')) + error_count = len(results) - success_count + total_added = sum(r.get('added', 0) for r in results if isinstance(r, dict)) + total_removed = sum(r.get('removed', 0) for r in results if isinstance(r, dict)) + + print(f"Success: {success_count}/{len(results)}") + print(f"Errors: {error_count}/{len(results)}") + print(f"Total IPs Added: {total_added}") + print(f"Total IPs Removed: {total_removed}") + print(f"{'='*60}\n") + + return [r for r in results if isinstance(r, dict)] + + +async def main(): + """Main entry point for list fetcher""" + database_url = os.getenv('DATABASE_URL') + + if not database_url: + print("ERROR: DATABASE_URL environment variable not set") + return 1 + + fetcher = ListFetcher(database_url) + + try: + # Fetch and sync all lists + await fetcher.fetch_all_lists() + + # Run merge logic to sync detections with blacklist/whitelist priority + print("\n" + "="*60) + print("RUNNING MERGE LOGIC") + print("="*60 + "\n") + + # Import merge logic (avoid circular imports) + import sys + from pathlib import Path + merge_logic_path = Path(__file__).parent.parent + sys.path.insert(0, str(merge_logic_path)) + from merge_logic import MergeLogic + + merge = MergeLogic(database_url) + stats = merge.sync_public_blacklist_detections() + + print(f"\nMerge Logic Stats:") + print(f" Created detections: {stats['created']}") + print(f" Cleaned invalid detections: {stats['cleaned']}") + print(f" Skipped (whitelisted): {stats['skipped_whitelisted']}") + print("="*60 + "\n") + + return 0 + except Exception as e: + print(f"FATAL ERROR: {e}") + import traceback + traceback.print_exc() + return 1 + + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + sys.exit(exit_code) diff --git a/python_ml/list_fetcher/parsers.py b/python_ml/list_fetcher/parsers.py new file mode 100644 index 0000000..7e63f3a --- /dev/null +++ b/python_ml/list_fetcher/parsers.py @@ -0,0 +1,280 @@ +import re +import json +from typing import List, Dict, Set, Optional +from datetime import datetime +import ipaddress + + +class ListParser: + """Base parser for public IP lists""" + + @staticmethod + def validate_ip(ip_str: str) -> bool: + """Validate IP address or CIDR range""" + try: + ipaddress.ip_network(ip_str, strict=False) + return True + except ValueError: + return False + + @staticmethod + def normalize_cidr(ip_str: str) -> tuple[str, Optional[str]]: + """ + Normalize IP/CIDR to (ip_address, cidr_range) + Example: '1.2.3.0/24' -> ('1.2.3.0', '1.2.3.0/24') + '1.2.3.4' -> ('1.2.3.4', None) + """ + try: + network = ipaddress.ip_network(ip_str, strict=False) + if '/' in ip_str: + return (str(network.network_address), str(network)) + else: + return (ip_str, None) + except ValueError: + return (ip_str, None) + + +class SpamhausParser(ListParser): + """Parser for Spamhaus DROP list""" + + @staticmethod + def parse(content: str) -> Set[tuple[str, Optional[str]]]: + """ + Parse Spamhaus DROP format: + ; Comment lines start with semicolon + 1.2.3.0/24 ; SBL12345 + """ + ips = set() + lines = content.strip().split('\n') + + for line in lines: + line = line.strip() + + # Skip comments and empty lines + if not line or line.startswith(';') or line.startswith('#'): + continue + + # Extract IP/CIDR before comment + parts = line.split(';') + if parts: + ip_part = parts[0].strip() + if ip_part and ListParser.validate_ip(ip_part): + ips.add(ListParser.normalize_cidr(ip_part)) + + return ips + + +class TalosParser(ListParser): + """Parser for Talos Intelligence blacklist""" + + @staticmethod + def parse(content: str) -> Set[tuple[str, Optional[str]]]: + """ + Parse Talos format (plain IP list): + 1.2.3.4 + 5.6.7.0/24 + """ + ips = set() + lines = content.strip().split('\n') + + for line in lines: + line = line.strip() + + # Skip comments and empty lines + if not line or line.startswith('#') or line.startswith('//'): + continue + + # Validate and add + if ListParser.validate_ip(line): + ips.add(ListParser.normalize_cidr(line)) + + return ips + + +class AWSParser(ListParser): + """Parser for AWS IP ranges JSON""" + + @staticmethod + def parse(content: str) -> Set[tuple[str, Optional[str]]]: + """ + Parse AWS JSON format: + { + "prefixes": [ + {"ip_prefix": "1.2.3.0/24", "region": "us-east-1", "service": "EC2"} + ] + } + """ + ips = set() + + try: + data = json.loads(content) + + # IPv4 prefixes + for prefix in data.get('prefixes', []): + ip_prefix = prefix.get('ip_prefix') + if ip_prefix and ListParser.validate_ip(ip_prefix): + ips.add(ListParser.normalize_cidr(ip_prefix)) + + # IPv6 prefixes (optional) + for prefix in data.get('ipv6_prefixes', []): + ipv6_prefix = prefix.get('ipv6_prefix') + if ipv6_prefix and ListParser.validate_ip(ipv6_prefix): + ips.add(ListParser.normalize_cidr(ipv6_prefix)) + + except json.JSONDecodeError: + pass + + return ips + + +class GCPParser(ListParser): + """Parser for Google Cloud IP ranges JSON""" + + @staticmethod + def parse(content: str) -> Set[tuple[str, Optional[str]]]: + """ + Parse GCP JSON format: + { + "prefixes": [ + {"ipv4Prefix": "1.2.3.0/24"}, + {"ipv6Prefix": "2001:db8::/32"} + ] + } + """ + ips = set() + + try: + data = json.loads(content) + + for prefix in data.get('prefixes', []): + # IPv4 + ipv4 = prefix.get('ipv4Prefix') + if ipv4 and ListParser.validate_ip(ipv4): + ips.add(ListParser.normalize_cidr(ipv4)) + + # IPv6 + ipv6 = prefix.get('ipv6Prefix') + if ipv6 and ListParser.validate_ip(ipv6): + ips.add(ListParser.normalize_cidr(ipv6)) + + except json.JSONDecodeError: + pass + + return ips + + +class CloudflareParser(ListParser): + """Parser for Cloudflare IP list""" + + @staticmethod + def parse(content: str) -> Set[tuple[str, Optional[str]]]: + """ + Parse Cloudflare format (plain CIDR list): + 1.2.3.0/24 + 5.6.7.0/24 + """ + ips = set() + lines = content.strip().split('\n') + + for line in lines: + line = line.strip() + + # Skip empty lines and comments + if not line or line.startswith('#'): + continue + + if ListParser.validate_ip(line): + ips.add(ListParser.normalize_cidr(line)) + + return ips + + +class IANAParser(ListParser): + """Parser for IANA Root Servers""" + + @staticmethod + def parse(content: str) -> Set[tuple[str, Optional[str]]]: + """ + Parse IANA root servers (extract IPs from HTML/text) + Look for IPv4 addresses in format XXX.XXX.XXX.XXX + """ + ips = set() + + # Regex for IPv4 addresses + ipv4_pattern = r'\b(?:[0-9]{1,3}\.){3}[0-9]{1,3}\b' + matches = re.findall(ipv4_pattern, content) + + for ip in matches: + if ListParser.validate_ip(ip): + ips.add(ListParser.normalize_cidr(ip)) + + return ips + + +class NTPPoolParser(ListParser): + """Parser for NTP Pool servers""" + + @staticmethod + def parse(content: str) -> Set[tuple[str, Optional[str]]]: + """ + Parse NTP pool format (plain IP list or JSON) + Tries multiple formats + """ + ips = set() + + # Try JSON first + try: + data = json.loads(content) + if isinstance(data, list): + for item in data: + if isinstance(item, str) and ListParser.validate_ip(item): + ips.add(ListParser.normalize_cidr(item)) + elif isinstance(item, dict): + ip = item.get('ip') or item.get('address') + if ip and ListParser.validate_ip(ip): + ips.add(ListParser.normalize_cidr(ip)) + except json.JSONDecodeError: + # Fallback to plain text parsing + lines = content.strip().split('\n') + for line in lines: + line = line.strip() + if line and ListParser.validate_ip(line): + ips.add(ListParser.normalize_cidr(line)) + + return ips + + +# Parser registry +PARSERS: Dict[str, type[ListParser]] = { + 'spamhaus': SpamhausParser, + 'talos': TalosParser, + 'aws': AWSParser, + 'gcp': GCPParser, + 'cloudflare': CloudflareParser, + 'iana': IANAParser, + 'ntp': NTPPoolParser, +} + + +def get_parser(list_name: str) -> Optional[type[ListParser]]: + """Get parser by list name (case-insensitive match)""" + list_name_lower = list_name.lower() + + for key, parser in PARSERS.items(): + if key in list_name_lower: + return parser + + # Default fallback: try plain text parser + return TalosParser + + +def parse_list(list_name: str, content: str) -> Set[tuple[str, Optional[str]]]: + """ + Parse list content using appropriate parser + Returns set of (ip_address, cidr_range) tuples + """ + parser_class = get_parser(list_name) + if parser_class: + parser = parser_class() + return parser.parse(content) + return set() diff --git a/python_ml/list_fetcher/run_fetcher.py b/python_ml/list_fetcher/run_fetcher.py new file mode 100644 index 0000000..391ec70 --- /dev/null +++ b/python_ml/list_fetcher/run_fetcher.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +""" +IDS List Fetcher Runner +Fetches and syncs public blacklist/whitelist sources every 10 minutes +""" +import asyncio +import sys +import os + +# Add parent directory to path +sys.path.append(os.path.dirname(os.path.dirname(__file__))) + +from list_fetcher.fetcher import main + +if __name__ == "__main__": + exit_code = asyncio.run(main()) + sys.exit(exit_code) diff --git a/python_ml/list_fetcher/seed_lists.py b/python_ml/list_fetcher/seed_lists.py new file mode 100644 index 0000000..a996535 --- /dev/null +++ b/python_ml/list_fetcher/seed_lists.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +""" +Seed default public lists into database +Run after migration 006 to populate initial lists +""" +import psycopg2 +import os +import sys +import argparse + +# Add parent directory to path +sys.path.append(os.path.dirname(os.path.dirname(__file__))) + +from list_fetcher.fetcher import ListFetcher +import asyncio + + +DEFAULT_LISTS = [ + # Blacklists + { + 'name': 'Spamhaus DROP', + 'type': 'blacklist', + 'url': 'https://www.spamhaus.org/drop/drop.txt', + 'enabled': True, + 'fetch_interval_minutes': 10 + }, + { + 'name': 'Talos Intelligence IP Blacklist', + 'type': 'blacklist', + 'url': 'https://talosintelligence.com/documents/ip-blacklist', + 'enabled': False, # Disabled by default - verify URL first + 'fetch_interval_minutes': 10 + }, + + # Whitelists + { + 'name': 'AWS IP Ranges', + 'type': 'whitelist', + 'url': 'https://ip-ranges.amazonaws.com/ip-ranges.json', + 'enabled': True, + 'fetch_interval_minutes': 10 + }, + { + 'name': 'Google Cloud IP Ranges', + 'type': 'whitelist', + 'url': 'https://www.gstatic.com/ipranges/cloud.json', + 'enabled': True, + 'fetch_interval_minutes': 10 + }, + { + 'name': 'Cloudflare IPv4', + 'type': 'whitelist', + 'url': 'https://www.cloudflare.com/ips-v4', + 'enabled': True, + 'fetch_interval_minutes': 10 + }, + { + 'name': 'IANA Root Servers', + 'type': 'whitelist', + 'url': 'https://www.iana.org/domains/root/servers', + 'enabled': True, + 'fetch_interval_minutes': 10 + }, + { + 'name': 'NTP Pool Servers', + 'type': 'whitelist', + 'url': 'https://www.ntppool.org/zone/@', + 'enabled': False, # Disabled by default - zone parameter needed + 'fetch_interval_minutes': 10 + } +] + + +def seed_lists(database_url: str, dry_run: bool = False): + """Insert default lists into database""" + conn = psycopg2.connect(database_url) + + try: + with conn.cursor() as cur: + # Check if lists already exist + cur.execute("SELECT COUNT(*) FROM public_lists") + result = cur.fetchone() + existing_count = result[0] if result else 0 + + if existing_count > 0 and not dry_run: + print(f"⚠️ Warning: {existing_count} lists already exist in database") + response = input("Continue and add default lists? (y/n): ") + if response.lower() != 'y': + print("Aborted") + return + + print(f"\n{'='*60}") + print("SEEDING DEFAULT PUBLIC LISTS") + print(f"{'='*60}\n") + + for list_config in DEFAULT_LISTS: + if dry_run: + status = "✓ ENABLED" if list_config['enabled'] else "○ DISABLED" + print(f"{status} {list_config['type'].upper()}: {list_config['name']}") + print(f" URL: {list_config['url']}") + print() + else: + cur.execute(""" + INSERT INTO public_lists (name, type, url, enabled, fetch_interval_minutes) + VALUES (%s, %s, %s, %s, %s) + RETURNING id, name + """, ( + list_config['name'], + list_config['type'], + list_config['url'], + list_config['enabled'], + list_config['fetch_interval_minutes'] + )) + + result = cur.fetchone() + if result: + list_id, list_name = result + status = "✓" if list_config['enabled'] else "○" + print(f"{status} Added: {list_name} (ID: {list_id})") + + if not dry_run: + conn.commit() + print(f"\n✓ Successfully seeded {len(DEFAULT_LISTS)} lists") + print(f"{'='*60}\n") + else: + print(f"\n{'='*60}") + print(f"DRY RUN: Would seed {len(DEFAULT_LISTS)} lists") + print(f"{'='*60}\n") + + except Exception as e: + conn.rollback() + print(f"✗ Error: {e}") + import traceback + traceback.print_exc() + return 1 + finally: + conn.close() + + return 0 + + +async def sync_lists(database_url: str): + """Run initial sync of all enabled lists""" + print("\nRunning initial sync of enabled lists...\n") + fetcher = ListFetcher(database_url) + await fetcher.fetch_all_lists() + + +def main(): + parser = argparse.ArgumentParser(description='Seed default public lists') + parser.add_argument('--dry-run', action='store_true', help='Show what would be added without inserting') + parser.add_argument('--sync', action='store_true', help='Run initial sync after seeding') + args = parser.parse_args() + + database_url = os.getenv('DATABASE_URL') + if not database_url: + print("ERROR: DATABASE_URL environment variable not set") + return 1 + + # Seed lists + exit_code = seed_lists(database_url, dry_run=args.dry_run) + + if exit_code != 0: + return exit_code + + # Optionally sync + if args.sync and not args.dry_run: + asyncio.run(sync_lists(database_url)) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python_ml/merge_logic.py b/python_ml/merge_logic.py new file mode 100644 index 0000000..a73de19 --- /dev/null +++ b/python_ml/merge_logic.py @@ -0,0 +1,351 @@ +#!/usr/bin/env python3 +""" +Merge Logic for Public Lists Integration +Implements priority: Manual Whitelist > Public Whitelist > Public Blacklist +""" +import os +import psycopg2 +from typing import Dict, Set, Optional +from datetime import datetime +import logging +import ipaddress + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) + + +def ip_matches_cidr(ip_address: str, cidr_range: Optional[str]) -> bool: + """ + Check if IP address matches CIDR range + Returns True if cidr_range is None (exact match) or if IP is in range + """ + if not cidr_range: + return True # Exact match handling + + try: + ip = ipaddress.ip_address(ip_address) + network = ipaddress.ip_network(cidr_range, strict=False) + return ip in network + except (ValueError, TypeError): + logger.warning(f"Invalid IP/CIDR: {ip_address}/{cidr_range}") + return False + + +class MergeLogic: + """ + Handles merge logic between manual entries and public lists + Priority: Manual whitelist > Public whitelist > Public blacklist + """ + + def __init__(self, database_url: str): + self.database_url = database_url + + def get_db_connection(self): + """Create database connection""" + return psycopg2.connect(self.database_url) + + def get_all_whitelisted_ips(self) -> Set[str]: + """ + Get all whitelisted IPs (manual + public) + Manual whitelist has higher priority than public whitelist + """ + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT DISTINCT ip_address + FROM whitelist + WHERE active = true + """) + return {row[0] for row in cur.fetchall()} + finally: + conn.close() + + def get_public_blacklist_ips(self) -> Set[str]: + """Get all active public blacklist IPs""" + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + cur.execute(""" + SELECT DISTINCT ip_address + FROM public_blacklist_ips + WHERE is_active = true + """) + return {row[0] for row in cur.fetchall()} + finally: + conn.close() + + def should_block_ip(self, ip_address: str) -> tuple[bool, str]: + """ + Determine if IP should be blocked based on merge logic + Returns: (should_block, reason) + + Priority: + 1. Manual whitelist (exact or CIDR) → DON'T block (highest priority) + 2. Public whitelist (exact or CIDR) → DON'T block + 3. Public blacklist (exact or CIDR) → DO block + 4. Not in any list → DON'T block (only ML decides) + """ + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + # Check manual whitelist (highest priority) - exact + CIDR matching + cur.execute(""" + SELECT ip_address, list_id FROM whitelist + WHERE active = true + AND source = 'manual' + """) + for row in cur.fetchall(): + wl_ip, wl_cidr = row[0], None + # Check if whitelist entry has CIDR notation + if '/' in wl_ip: + wl_cidr = wl_ip + if wl_ip == ip_address or ip_matches_cidr(ip_address, wl_cidr): + return (False, "manual_whitelist") + + # Check public whitelist (any source except 'manual') - exact + CIDR + cur.execute(""" + SELECT ip_address, list_id FROM whitelist + WHERE active = true + AND source != 'manual' + """) + for row in cur.fetchall(): + wl_ip, wl_cidr = row[0], None + if '/' in wl_ip: + wl_cidr = wl_ip + if wl_ip == ip_address or ip_matches_cidr(ip_address, wl_cidr): + return (False, "public_whitelist") + + # Check public blacklist - exact + CIDR matching + cur.execute(""" + SELECT id, ip_address, cidr_range FROM public_blacklist_ips + WHERE is_active = true + """) + for row in cur.fetchall(): + bl_id, bl_ip, bl_cidr = row + # Match exact IP or check if IP is in CIDR range + if bl_ip == ip_address or ip_matches_cidr(ip_address, bl_cidr): + return (True, f"public_blacklist:{bl_id}") + + # Not in any list + return (False, "not_listed") + finally: + conn.close() + + def create_detection_from_blacklist( + self, + ip_address: str, + blacklist_id: str, + risk_score: int = 75 + ) -> Optional[str]: + """ + Create detection record for public blacklist IP + Only if not whitelisted (priority check) + """ + should_block, reason = self.should_block_ip(ip_address) + + if not should_block: + logger.info(f"IP {ip_address} not blocked - reason: {reason}") + return None + + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + # Check if detection already exists + cur.execute(""" + SELECT id FROM detections + WHERE source_ip = %s + AND detection_source = 'public_blacklist' + LIMIT 1 + """, (ip_address,)) + + existing = cur.fetchone() + if existing: + logger.info(f"Detection already exists for {ip_address}") + return existing[0] + + # Create new detection + cur.execute(""" + INSERT INTO detections ( + source_ip, + risk_score, + anomaly_type, + detection_source, + blacklist_id, + detected_at, + blocked + ) VALUES (%s, %s, %s, %s, %s, %s, %s) + RETURNING id + """, ( + ip_address, + str(risk_score), + 'public_blacklist', + 'public_blacklist', + blacklist_id, + datetime.utcnow(), + False # Will be blocked by auto-block service if risk_score >= 80 + )) + + result = cur.fetchone() + if not result: + logger.error(f"Failed to get detection ID after insert for {ip_address}") + return None + + detection_id = result[0] + conn.commit() + + logger.info(f"Created detection {detection_id} for blacklisted IP {ip_address}") + return detection_id + except Exception as e: + conn.rollback() + logger.error(f"Failed to create detection for {ip_address}: {e}") + return None + finally: + conn.close() + + def cleanup_invalid_detections(self) -> int: + """ + Remove detections for IPs that are now whitelisted + Respects priority: manual/public whitelist overrides blacklist + """ + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + # Delete detections for whitelisted IPs + cur.execute(""" + DELETE FROM detections + WHERE detection_source = 'public_blacklist' + AND source_ip IN ( + SELECT ip_address FROM whitelist WHERE active = true + ) + """) + deleted = cur.rowcount + conn.commit() + + if deleted > 0: + logger.info(f"Cleaned up {deleted} detections for whitelisted IPs") + + return deleted + except Exception as e: + conn.rollback() + logger.error(f"Failed to cleanup detections: {e}") + return 0 + finally: + conn.close() + + def sync_public_blacklist_detections(self) -> Dict[str, int]: + """ + Sync detections with current public blacklist state using BULK operations + Creates detections for blacklisted IPs (if not whitelisted) + Removes detections for IPs no longer blacklisted or now whitelisted + """ + stats = { + 'created': 0, + 'cleaned': 0, + 'skipped_whitelisted': 0 + } + + conn = self.get_db_connection() + try: + with conn.cursor() as cur: + # Cleanup whitelisted IPs first (priority) + stats['cleaned'] = self.cleanup_invalid_detections() + + # Bulk create detections for blacklisted IPs (excluding whitelisted) + # Uses PostgreSQL INET/CIDR operators for proper CIDR range matching + # Critical for performance with 186M+ rows (single query vs O(N) loops) + cur.execute(""" + WITH blacklisted_ranges AS ( + -- Blacklist entries with CIDR ranges (e.g. Spamhaus /24) + SELECT + bl.id as blacklist_id, + bl.ip_address, + COALESCE(bl.cidr_range, bl.ip_address) as cidr + FROM public_blacklist_ips bl + WHERE bl.is_active = true + ), + whitelisted_ranges AS ( + -- Whitelist entries (manual + public) with CIDR support + SELECT + ip_address, + CASE + WHEN ip_address ~ '/' THEN ip_address::inet + ELSE ip_address::inet + END as ip_range, + source + FROM whitelist + WHERE active = true + ) + INSERT INTO detections ( + source_ip, + risk_score, + anomaly_type, + detection_source, + blacklist_id, + detected_at, + blocked + ) + SELECT DISTINCT + bl.ip_address, + '75', + 'public_blacklist', + 'public_blacklist', + bl.blacklist_id, + NOW(), + false + FROM blacklisted_ranges bl + WHERE bl.cidr IS NOT NULL + -- Exclude if IP is in any whitelist range (manual or public) + -- Priority: Manual whitelist > Public whitelist > Blacklist + AND NOT EXISTS ( + SELECT 1 FROM whitelisted_ranges wl + WHERE bl.ip_address::inet <<= wl.ip_range + ) + -- Avoid duplicate detections + AND NOT EXISTS ( + SELECT 1 FROM detections d + WHERE d.source_ip = bl.ip_address + AND d.detection_source = 'public_blacklist' + ) + RETURNING id + """) + + created_ids = cur.fetchall() + stats['created'] = len(created_ids) + conn.commit() + + logger.info(f"Bulk sync complete: {stats}") + return stats + except Exception as e: + conn.rollback() + logger.error(f"Failed to sync detections: {e}") + import traceback + traceback.print_exc() + return stats + finally: + conn.close() + + +def main(): + """Run merge logic sync""" + database_url = os.environ.get('DATABASE_URL') + if not database_url: + logger.error("DATABASE_URL environment variable not set") + return 1 + + merge = MergeLogic(database_url) + stats = merge.sync_public_blacklist_detections() + + print(f"\n{'='*60}") + print("MERGE LOGIC SYNC COMPLETED") + print(f"{'='*60}") + print(f"Created detections: {stats['created']}") + print(f"Cleaned invalid detections: {stats['cleaned']}") + print(f"Skipped (whitelisted): {stats['skipped_whitelisted']}") + print(f"{'='*60}\n") + + return 0 + + +if __name__ == "__main__": + exit(main()) diff --git a/server/routes.ts b/server/routes.ts index 95bfad1..78f837b 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -1,7 +1,7 @@ import type { Express } from "express"; import { createServer, type Server } from "http"; import { storage } from "./storage"; -import { insertRouterSchema, insertDetectionSchema, insertWhitelistSchema, networkAnalytics, routers } from "@shared/schema"; +import { insertRouterSchema, insertDetectionSchema, insertWhitelistSchema, insertPublicListSchema, networkAnalytics, routers } from "@shared/schema"; import { db } from "./db"; import { desc, eq } from "drizzle-orm"; @@ -148,6 +148,127 @@ export async function registerRoutes(app: Express): Promise { } }); + // Public Lists + app.get("/api/public-lists", async (req, res) => { + try { + const lists = await storage.getAllPublicLists(); + res.json(lists); + } catch (error) { + console.error('[DB ERROR] Failed to fetch public lists:', error); + res.status(500).json({ error: "Failed to fetch public lists" }); + } + }); + + app.get("/api/public-lists/:id", async (req, res) => { + try { + const list = await storage.getPublicListById(req.params.id); + if (!list) { + return res.status(404).json({ error: "List not found" }); + } + res.json(list); + } catch (error) { + res.status(500).json({ error: "Failed to fetch list" }); + } + }); + + app.post("/api/public-lists", async (req, res) => { + try { + const validatedData = insertPublicListSchema.parse(req.body); + const list = await storage.createPublicList(validatedData); + res.json(list); + } catch (error: any) { + console.error('[API ERROR] Failed to create public list:', error); + if (error.name === 'ZodError') { + return res.status(400).json({ error: "Invalid list data", details: error.errors }); + } + res.status(400).json({ error: "Invalid list data" }); + } + }); + + app.patch("/api/public-lists/:id", async (req, res) => { + try { + const validatedData = insertPublicListSchema.partial().parse(req.body); + const list = await storage.updatePublicList(req.params.id, validatedData); + if (!list) { + return res.status(404).json({ error: "List not found" }); + } + res.json(list); + } catch (error: any) { + console.error('[API ERROR] Failed to update public list:', error); + if (error.name === 'ZodError') { + return res.status(400).json({ error: "Invalid list data", details: error.errors }); + } + res.status(400).json({ error: "Invalid list data" }); + } + }); + + app.delete("/api/public-lists/:id", async (req, res) => { + try { + const success = await storage.deletePublicList(req.params.id); + if (!success) { + return res.status(404).json({ error: "List not found" }); + } + res.json({ success: true }); + } catch (error) { + res.status(500).json({ error: "Failed to delete list" }); + } + }); + + app.post("/api/public-lists/:id/sync", async (req, res) => { + try { + const list = await storage.getPublicListById(req.params.id); + if (!list) { + return res.status(404).json({ error: "List not found" }); + } + + const updated = await storage.updatePublicList(req.params.id, { + lastAttempt: new Date('1970-01-01T00:00:00Z'), + errorMessage: null, + }); + + res.json({ + success: true, + message: "Manual sync triggered - list marked for immediate sync", + note: "Fetcher will sync this list on next cycle (max 10 minutes). Check logs: journalctl -u ids-list-fetcher -n 50", + list: updated + }); + } catch (error) { + console.error('[API ERROR] Failed to trigger sync:', error); + res.status(500).json({ error: "Failed to trigger sync" }); + } + }); + + // Public Blacklist IPs + app.get("/api/public-blacklist", async (req, res) => { + try { + const limit = parseInt(req.query.limit as string) || 1000; + const listId = req.query.listId as string | undefined; + const ipAddress = req.query.ipAddress as string | undefined; + const isActive = req.query.isActive === 'true'; + + const ips = await storage.getPublicBlacklistIps({ + limit, + listId, + ipAddress, + isActive: req.query.isActive !== undefined ? isActive : undefined, + }); + res.json(ips); + } catch (error) { + console.error('[DB ERROR] Failed to fetch blacklist IPs:', error); + res.status(500).json({ error: "Failed to fetch blacklist IPs" }); + } + }); + + app.get("/api/public-blacklist/stats", async (req, res) => { + try { + const stats = await storage.getPublicBlacklistStats(); + res.json(stats); + } catch (error) { + console.error('[DB ERROR] Failed to fetch blacklist stats:', error); + res.status(500).json({ error: "Failed to fetch stats" }); + } + }); + // Training History app.get("/api/training-history", async (req, res) => { try { diff --git a/server/storage.ts b/server/storage.ts index b7a1bd7..9b19923 100644 --- a/server/storage.ts +++ b/server/storage.ts @@ -5,6 +5,8 @@ import { whitelist, trainingHistory, networkAnalytics, + publicLists, + publicBlacklistIps, type Router, type InsertRouter, type NetworkLog, @@ -16,6 +18,10 @@ import { type TrainingHistory, type InsertTrainingHistory, type NetworkAnalytics, + type PublicList, + type InsertPublicList, + type PublicBlacklistIp, + type InsertPublicBlacklistIp, } from "@shared/schema"; import { db } from "./db"; import { eq, desc, and, gte, sql, inArray } from "drizzle-orm"; @@ -74,6 +80,26 @@ export interface IStorage { recentDetections: Detection[]; }>; + // Public Lists + getAllPublicLists(): Promise; + getPublicListById(id: string): Promise; + createPublicList(list: InsertPublicList): Promise; + updatePublicList(id: string, list: Partial): Promise; + deletePublicList(id: string): Promise; + + // Public Blacklist IPs + getPublicBlacklistIps(options: { + limit?: number; + listId?: string; + ipAddress?: string; + isActive?: boolean; + }): Promise; + getPublicBlacklistStats(): Promise<{ + totalLists: number; + totalIps: number; + overlapWithDetections: number; + }>; + // System testConnection(): Promise; } @@ -387,6 +413,107 @@ export class DatabaseStorage implements IStorage { }; } + // Public Lists + async getAllPublicLists(): Promise { + return await db.select().from(publicLists).orderBy(desc(publicLists.createdAt)); + } + + async getPublicListById(id: string): Promise { + const [list] = await db.select().from(publicLists).where(eq(publicLists.id, id)); + return list || undefined; + } + + async createPublicList(insertList: InsertPublicList): Promise { + const [list] = await db.insert(publicLists).values(insertList).returning(); + return list; + } + + async updatePublicList(id: string, updateData: Partial): Promise { + const [list] = await db + .update(publicLists) + .set(updateData) + .where(eq(publicLists.id, id)) + .returning(); + return list || undefined; + } + + async deletePublicList(id: string): Promise { + const result = await db.delete(publicLists).where(eq(publicLists.id, id)); + return result.rowCount !== null && result.rowCount > 0; + } + + // Public Blacklist IPs + async getPublicBlacklistIps(options: { + limit?: number; + listId?: string; + ipAddress?: string; + isActive?: boolean; + }): Promise { + const { limit = 1000, listId, ipAddress, isActive } = options; + + const conditions = []; + + if (listId) { + conditions.push(eq(publicBlacklistIps.listId, listId)); + } + + if (ipAddress) { + conditions.push(eq(publicBlacklistIps.ipAddress, ipAddress)); + } + + if (isActive !== undefined) { + conditions.push(eq(publicBlacklistIps.isActive, isActive)); + } + + const query = db + .select() + .from(publicBlacklistIps) + .orderBy(desc(publicBlacklistIps.lastSeen)) + .limit(limit); + + if (conditions.length > 0) { + return await query.where(and(...conditions)); + } + + return await query; + } + + async getPublicBlacklistStats(): Promise<{ + totalLists: number; + totalIps: number; + overlapWithDetections: number; + }> { + const lists = await db.select().from(publicLists).where(eq(publicLists.type, 'blacklist')); + const totalLists = lists.length; + + const [{ count: totalIps }] = await db + .select({ count: sql`count(*)::int` }) + .from(publicBlacklistIps) + .where(eq(publicBlacklistIps.isActive, true)); + + const [{ count: overlapWithDetections }] = await db + .select({ count: sql`count(distinct ${detections.sourceIp})::int` }) + .from(detections) + .innerJoin(publicBlacklistIps, eq(detections.sourceIp, publicBlacklistIps.ipAddress)) + .where( + and( + eq(publicBlacklistIps.isActive, true), + eq(detections.detectionSource, 'public_blacklist'), + sql`NOT EXISTS ( + SELECT 1 FROM ${whitelist} + WHERE ${whitelist.ipAddress} = ${detections.sourceIp} + AND ${whitelist.active} = true + )` + ) + ); + + return { + totalLists, + totalIps: totalIps || 0, + overlapWithDetections: overlapWithDetections || 0, + }; + } + async testConnection(): Promise { try { await db.execute(sql`SELECT 1`); diff --git a/shared/schema.ts b/shared/schema.ts index daf0d89..bbba683 100644 --- a/shared/schema.ts +++ b/shared/schema.ts @@ -58,11 +58,15 @@ export const detections = pgTable("detections", { asNumber: text("as_number"), asName: text("as_name"), isp: text("isp"), + // Public lists integration + detectionSource: text("detection_source").notNull().default("ml_model"), + blacklistId: varchar("blacklist_id").references(() => publicBlacklistIps.id, { onDelete: 'set null' }), }, (table) => ({ sourceIpIdx: index("detection_source_ip_idx").on(table.sourceIp), riskScoreIdx: index("risk_score_idx").on(table.riskScore), detectedAtIdx: index("detected_at_idx").on(table.detectedAt), countryIdx: index("country_idx").on(table.country), + detectionSourceIdx: index("detection_source_idx").on(table.detectionSource), })); // Whitelist per IP fidati @@ -74,7 +78,12 @@ export const whitelist = pgTable("whitelist", { createdBy: text("created_by"), active: boolean("active").notNull().default(true), createdAt: timestamp("created_at").defaultNow().notNull(), -}); + // Public lists integration + source: text("source").notNull().default("manual"), + listId: varchar("list_id").references(() => publicLists.id, { onDelete: 'set null' }), +}, (table) => ({ + sourceIdx: index("whitelist_source_idx").on(table.source), +})); // ML Training history export const trainingHistory = pgTable("training_history", { @@ -125,6 +134,42 @@ export const networkAnalytics = pgTable("network_analytics", { dateHourUnique: unique("network_analytics_date_hour_key").on(table.date, table.hour), })); +// Public threat/whitelist sources +export const publicLists = pgTable("public_lists", { + id: varchar("id").primaryKey().default(sql`gen_random_uuid()`), + name: text("name").notNull(), + type: text("type").notNull(), + url: text("url").notNull(), + enabled: boolean("enabled").notNull().default(true), + fetchIntervalMinutes: integer("fetch_interval_minutes").notNull().default(10), + lastFetch: timestamp("last_fetch"), + lastSuccess: timestamp("last_success"), + totalIps: integer("total_ips").notNull().default(0), + activeIps: integer("active_ips").notNull().default(0), + errorCount: integer("error_count").notNull().default(0), + lastError: text("last_error"), + createdAt: timestamp("created_at").defaultNow().notNull(), +}, (table) => ({ + typeIdx: index("public_lists_type_idx").on(table.type), + enabledIdx: index("public_lists_enabled_idx").on(table.enabled), +})); + +// Public blacklist IPs from external sources +export const publicBlacklistIps = pgTable("public_blacklist_ips", { + id: varchar("id").primaryKey().default(sql`gen_random_uuid()`), + ipAddress: text("ip_address").notNull(), + cidrRange: text("cidr_range"), + listId: varchar("list_id").notNull().references(() => publicLists.id, { onDelete: 'cascade' }), + firstSeen: timestamp("first_seen").defaultNow().notNull(), + lastSeen: timestamp("last_seen").defaultNow().notNull(), + isActive: boolean("is_active").notNull().default(true), +}, (table) => ({ + ipAddressIdx: index("public_blacklist_ip_idx").on(table.ipAddress), + listIdIdx: index("public_blacklist_list_idx").on(table.listId), + isActiveIdx: index("public_blacklist_active_idx").on(table.isActive), + ipListUnique: unique("public_blacklist_ip_list_key").on(table.ipAddress, table.listId), +})); + // Schema version tracking for database migrations export const schemaVersion = pgTable("schema_version", { id: integer("id").primaryKey().default(1), @@ -138,7 +183,30 @@ export const routersRelations = relations(routers, ({ many }) => ({ logs: many(networkLogs), })); -// Rimossa relazione router (non più FK) +export const publicListsRelations = relations(publicLists, ({ many }) => ({ + blacklistIps: many(publicBlacklistIps), +})); + +export const publicBlacklistIpsRelations = relations(publicBlacklistIps, ({ one }) => ({ + list: one(publicLists, { + fields: [publicBlacklistIps.listId], + references: [publicLists.id], + }), +})); + +export const whitelistRelations = relations(whitelist, ({ one }) => ({ + list: one(publicLists, { + fields: [whitelist.listId], + references: [publicLists.id], + }), +})); + +export const detectionsRelations = relations(detections, ({ one }) => ({ + blacklist: one(publicBlacklistIps, { + fields: [detections.blacklistId], + references: [publicBlacklistIps.id], + }), +})); // Insert schemas export const insertRouterSchema = createInsertSchema(routers).omit({ @@ -176,6 +244,19 @@ export const insertNetworkAnalyticsSchema = createInsertSchema(networkAnalytics) createdAt: true, }); +export const insertPublicListSchema = createInsertSchema(publicLists).omit({ + id: true, + createdAt: true, + lastFetch: true, + lastSuccess: true, +}); + +export const insertPublicBlacklistIpSchema = createInsertSchema(publicBlacklistIps).omit({ + id: true, + firstSeen: true, + lastSeen: true, +}); + // Types export type Router = typeof routers.$inferSelect; export type InsertRouter = z.infer; @@ -197,3 +278,9 @@ export type InsertSchemaVersion = z.infer; export type NetworkAnalytics = typeof networkAnalytics.$inferSelect; export type InsertNetworkAnalytics = z.infer; + +export type PublicList = typeof publicLists.$inferSelect; +export type InsertPublicList = z.infer; + +export type PublicBlacklistIp = typeof publicBlacklistIps.$inferSelect; +export type InsertPublicBlacklistIp = z.infer; diff --git a/uv.lock b/uv.lock new file mode 100644 index 0000000..1e086e8 --- /dev/null +++ b/uv.lock @@ -0,0 +1,101 @@ +version = 1 +revision = 3 +requires-python = ">=3.11" + +[[package]] +name = "anyio" +version = "4.11.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "sniffio" }, + { name = "typing-extensions", marker = "python_full_version < '3.13'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c6/78/7d432127c41b50bccba979505f272c16cbcadcc33645d5fa3a738110ae75/anyio-4.11.0.tar.gz", hash = "sha256:82a8d0b81e318cc5ce71a5f1f8b5c4e63619620b63141ef8c995fa0db95a57c4", size = 219094, upload-time = "2025-09-23T09:19:12.58Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/b3/9b1a8074496371342ec1e796a96f99c82c945a339cd81a8e73de28b4cf9e/anyio-4.11.0-py3-none-any.whl", hash = "sha256:0287e96f4d26d4149305414d4e3bc32f0dcd0862365a4bddea19d7a1ec38c4fc", size = 109097, upload-time = "2025-09-23T09:19:10.601Z" }, +] + +[[package]] +name = "certifi" +version = "2025.11.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/8c/58f469717fa48465e4a50c014a0400602d3c437d7c0c468e17ada824da3a/certifi-2025.11.12.tar.gz", hash = "sha256:d8ab5478f2ecd78af242878415affce761ca6bc54a22a27e026d7c25357c3316", size = 160538, upload-time = "2025-11-12T02:54:51.517Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/70/7d/9bc192684cea499815ff478dfcdc13835ddf401365057044fb721ec6bddb/certifi-2025.11.12-py3-none-any.whl", hash = "sha256:97de8790030bbd5c2d96b7ec782fc2f7820ef8dba6db909ccf95449f2d062d4b", size = 159438, upload-time = "2025-11-12T02:54:49.735Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/6f/6d/0703ccc57f3a7233505399edb88de3cbd678da106337b9fcde432b65ed60/idna-3.11.tar.gz", hash = "sha256:795dafcc9c04ed0c1fb032c2aa73654d8e8c5023a7df64a53f39190ada629902", size = 194582, upload-time = "2025-10-12T14:55:20.501Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0e/61/66938bbb5fc52dbdf84594873d5b51fb1f7c7794e9c0f5bd885f30bc507b/idna-3.11-py3-none-any.whl", hash = "sha256:771a87f49d9defaf64091e6e6fe9c18d4833f140bd19464795bc32d966ca37ea", size = 71008, upload-time = "2025-10-12T14:55:18.883Z" }, +] + +[[package]] +name = "repl-nix-workspace" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "httpx" }, +] + +[package.metadata] +requires-dist = [{ name = "httpx", specifier = ">=0.28.1" }] + +[[package]] +name = "sniffio" +version = "1.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372, upload-time = "2024-02-25T23:20:04.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235, upload-time = "2024-02-25T23:20:01.196Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.15.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/72/94/1a15dd82efb362ac84269196e94cf00f187f7ed21c242792a923cdb1c61f/typing_extensions-4.15.0.tar.gz", hash = "sha256:0cea48d173cc12fa28ecabc3b837ea3cf6f38c6d1136f85cbaaf598984861466", size = 109391, upload-time = "2025-08-25T13:49:26.313Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, +]