From 3574ff02743a64bb1b8576d311e830f110b36361 Mon Sep 17 00:00:00 2001 From: marco370 <48531002-marco370@users.noreply.replit.com> Date: Fri, 2 Jan 2026 14:44:54 +0000 Subject: [PATCH] Update database schema and migrations to correctly handle IP address data types Introduce migration 008 to force INET and CIDR types for IP-related columns in `whitelist` and `public_blacklist_ips` tables, and update `shared/schema.ts` with comments clarifying production type handling. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: intermediate_checkpoint Replit-Commit-Event-Id: 1d0f629d-65cf-420d-86d9-a51b24caffa4 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/rDib6Pq --- ...n-50-no-pager-Jan-02-15-_1767364928312.txt | 51 ++++++++++ .../migrations/008_force_inet_types.sql | 92 +++++++++++++++++++ shared/schema.ts | 10 +- 3 files changed, 150 insertions(+), 3 deletions(-) create mode 100644 attached_assets/Pasted--journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-15-_1767364928312.txt create mode 100644 deployment/migrations/008_force_inet_types.sql diff --git a/attached_assets/Pasted--journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-15-_1767364928312.txt b/attached_assets/Pasted--journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-15-_1767364928312.txt new file mode 100644 index 0000000..806af30 --- /dev/null +++ b/attached_assets/Pasted--journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-15-_1767364928312.txt @@ -0,0 +1,51 @@ + journalctl -u ids-list-fetcher -n 50 --no-pager +Jan 02 15:30:01 ids.alfacom.it ids-list-fetcher[9296]: Skipped (whitelisted): 0 +Jan 02 15:30:01 ids.alfacom.it ids-list-fetcher[9296]: ============================================================ +Jan 02 15:30:01 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully. +Jan 02 15:30:01 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service. +Jan 02 15:40:00 ids.alfacom.it systemd[1]: Starting IDS Public Lists Fetcher Service... +Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: ============================================================ +Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: [2026-01-02 15:40:00] PUBLIC LISTS SYNC +Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: ============================================================ +Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: Found 2 enabled lists +Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: [15:40:00] Downloading Spamhaus from https://www.spamhaus.org/drop/drop_v4.json... +Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: [15:40:00] Downloading AWS from https://ip-ranges.amazonaws.com/ip-ranges.json... +Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: [15:40:00] Parsing AWS... +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] Found 9548 IPs, syncing to database... +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] ✓ AWS: +0 -0 ~9511 +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] Parsing Spamhaus... +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] Found 1468 IPs, syncing to database... +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] ✓ Spamhaus: +0 -0 ~1464 +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================ +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: SYNC SUMMARY +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================ +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Success: 2/2 +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Errors: 0/2 +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Total IPs Added: 0 +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Total IPs Removed: 0 +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================ +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================ +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: RUNNING MERGE LOGIC +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================ +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ERROR:merge_logic:Failed to cleanup detections: operator does not exist: inet = text +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: LINE 9: d.source_ip::inet = wl.ip_inet +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ^ +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts. +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ERROR:merge_logic:Failed to sync detections: operator does not exist: inet = text +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: LINE 29: bl.ip_inet = wl.ip_inet +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ^ +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts. +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Traceback (most recent call last): +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: File "/opt/ids/python_ml/merge_logic.py", line 264, in sync_public_blacklist_detections +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: cur.execute(""" +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: psycopg2.errors.UndefinedFunction: operator does not exist: inet = text +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: LINE 29: bl.ip_inet = wl.ip_inet +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ^ +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts. +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Merge Logic Stats: +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Created detections: 0 +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Cleaned invalid detections: 0 +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Skipped (whitelisted): 0 +Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================ +Jan 02 15:40:01 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully. +Jan 02 15:40:01 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service. \ No newline at end of file diff --git a/deployment/migrations/008_force_inet_types.sql b/deployment/migrations/008_force_inet_types.sql new file mode 100644 index 0000000..4686a78 --- /dev/null +++ b/deployment/migrations/008_force_inet_types.sql @@ -0,0 +1,92 @@ +-- Migration 008: Force INET/CIDR types (unconditional) +-- Fixes issues where columns remained TEXT after conditional migration 007 +-- Date: 2026-01-02 + +BEGIN; + +-- ============================================================================ +-- FORCE DROP AND RECREATE ALL INET COLUMNS +-- This is unconditional - always executes regardless of current state +-- ============================================================================ + +-- Drop indexes first (if exist) +DROP INDEX IF EXISTS public_blacklist_ip_inet_idx; +DROP INDEX IF EXISTS public_blacklist_cidr_inet_idx; +DROP INDEX IF EXISTS whitelist_ip_inet_idx; + +-- ============================================================================ +-- FIX public_blacklist_ips TABLE +-- ============================================================================ + +-- Drop columns unconditionally +ALTER TABLE public_blacklist_ips DROP COLUMN IF EXISTS ip_inet; +ALTER TABLE public_blacklist_ips DROP COLUMN IF EXISTS cidr_inet; + +-- Recreate with correct INET/CIDR types +ALTER TABLE public_blacklist_ips ADD COLUMN ip_inet inet; +ALTER TABLE public_blacklist_ips ADD COLUMN cidr_inet cidr; + +-- Populate from existing text data +UPDATE public_blacklist_ips +SET + ip_inet = CASE + WHEN ip_address ~ '/' THEN ip_address::inet + ELSE ip_address::inet + END, + cidr_inet = CASE + WHEN cidr_range IS NOT NULL AND cidr_range != '' THEN cidr_range::cidr + WHEN ip_address ~ '/' THEN ip_address::cidr + ELSE (ip_address || '/32')::cidr + END +WHERE ip_inet IS NULL; + +-- Create GiST indexes for fast INET/CIDR containment operators +CREATE INDEX public_blacklist_ip_inet_idx ON public_blacklist_ips USING gist(ip_inet inet_ops); +CREATE INDEX public_blacklist_cidr_inet_idx ON public_blacklist_ips USING gist(cidr_inet inet_ops); + +-- ============================================================================ +-- FIX whitelist TABLE +-- ============================================================================ + +-- Drop column unconditionally +ALTER TABLE whitelist DROP COLUMN IF EXISTS ip_inet; + +-- Recreate with correct INET type +ALTER TABLE whitelist ADD COLUMN ip_inet inet; + +-- Populate from existing text data +UPDATE whitelist +SET ip_inet = CASE + WHEN ip_address ~ '/' THEN ip_address::inet + ELSE ip_address::inet +END +WHERE ip_inet IS NULL; + +-- Create index for whitelist +CREATE INDEX whitelist_ip_inet_idx ON whitelist USING gist(ip_inet inet_ops); + +-- ============================================================================ +-- UPDATE SCHEMA VERSION +-- ============================================================================ + +UPDATE schema_version SET version = 8, applied_at = NOW() WHERE id = 1; + +COMMIT; + +-- ============================================================================ +-- VERIFICATION +-- ============================================================================ + +SELECT 'Migration 008 completed successfully' as status; +SELECT version, applied_at FROM schema_version WHERE id = 1; + +-- Verify column types +SELECT + table_name, + column_name, + data_type +FROM information_schema.columns +WHERE + (table_name = 'public_blacklist_ips' AND column_name IN ('ip_inet', 'cidr_inet')) + OR (table_name = 'whitelist' AND column_name = 'ip_inet') +ORDER BY table_name, column_name; diff --git a/shared/schema.ts b/shared/schema.ts index f617d94..f6841d1 100644 --- a/shared/schema.ts +++ b/shared/schema.ts @@ -70,10 +70,12 @@ export const detections = pgTable("detections", { })); // Whitelist per IP fidati +// NOTE: ip_inet is INET type in production (managed by SQL migrations) +// Drizzle lacks native INET support, so we use text() here export const whitelist = pgTable("whitelist", { id: varchar("id").primaryKey().default(sql`gen_random_uuid()`), ipAddress: text("ip_address").notNull().unique(), - ipInet: text("ip_inet"), + ipInet: text("ip_inet"), // Actually INET in production - see migration 008 comment: text("comment"), reason: text("reason"), createdBy: text("created_by"), @@ -156,12 +158,14 @@ export const publicLists = pgTable("public_lists", { })); // Public blacklist IPs from external sources +// NOTE: ip_inet/cidr_inet are INET/CIDR types in production (managed by SQL migrations) +// Drizzle lacks native INET/CIDR support, so we use text() here export const publicBlacklistIps = pgTable("public_blacklist_ips", { id: varchar("id").primaryKey().default(sql`gen_random_uuid()`), ipAddress: text("ip_address").notNull(), cidrRange: text("cidr_range"), - ipInet: text("ip_inet"), - cidrInet: text("cidr_inet"), + ipInet: text("ip_inet"), // Actually INET in production - see migration 008 + cidrInet: text("cidr_inet"), // Actually CIDR in production - see migration 008 listId: varchar("list_id").notNull().references(() => publicLists.id, { onDelete: 'cascade' }), firstSeen: timestamp("first_seen").defaultNow().notNull(), lastSeen: timestamp("last_seen").defaultNow().notNull(),