Update database schema and migrations to correctly handle IP address data types
Introduce migration 008 to force INET and CIDR types for IP-related columns in `whitelist` and `public_blacklist_ips` tables, and update `shared/schema.ts` with comments clarifying production type handling. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: intermediate_checkpoint Replit-Commit-Event-Id: 1d0f629d-65cf-420d-86d9-a51b24caffa4 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/rDib6Pq
This commit is contained in:
marco370
parent
0301a42825
commit
3574ff0274
@ -0,0 +1,51 @@
|
||||
journalctl -u ids-list-fetcher -n 50 --no-pager
|
||||
Jan 02 15:30:01 ids.alfacom.it ids-list-fetcher[9296]: Skipped (whitelisted): 0
|
||||
Jan 02 15:30:01 ids.alfacom.it ids-list-fetcher[9296]: ============================================================
|
||||
Jan 02 15:30:01 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully.
|
||||
Jan 02 15:30:01 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service.
|
||||
Jan 02 15:40:00 ids.alfacom.it systemd[1]: Starting IDS Public Lists Fetcher Service...
|
||||
Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: ============================================================
|
||||
Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: [2026-01-02 15:40:00] PUBLIC LISTS SYNC
|
||||
Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: ============================================================
|
||||
Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: Found 2 enabled lists
|
||||
Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: [15:40:00] Downloading Spamhaus from https://www.spamhaus.org/drop/drop_v4.json...
|
||||
Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: [15:40:00] Downloading AWS from https://ip-ranges.amazonaws.com/ip-ranges.json...
|
||||
Jan 02 15:40:00 ids.alfacom.it ids-list-fetcher[9493]: [15:40:00] Parsing AWS...
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] Found 9548 IPs, syncing to database...
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] ✓ AWS: +0 -0 ~9511
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] Parsing Spamhaus...
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] Found 1468 IPs, syncing to database...
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: [15:40:01] ✓ Spamhaus: +0 -0 ~1464
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: SYNC SUMMARY
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Success: 2/2
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Errors: 0/2
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Total IPs Added: 0
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Total IPs Removed: 0
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: RUNNING MERGE LOGIC
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ERROR:merge_logic:Failed to cleanup detections: operator does not exist: inet = text
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: LINE 9: d.source_ip::inet = wl.ip_inet
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ^
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts.
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ERROR:merge_logic:Failed to sync detections: operator does not exist: inet = text
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: LINE 29: bl.ip_inet = wl.ip_inet
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ^
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts.
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Traceback (most recent call last):
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: File "/opt/ids/python_ml/merge_logic.py", line 264, in sync_public_blacklist_detections
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: cur.execute("""
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: psycopg2.errors.UndefinedFunction: operator does not exist: inet = text
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: LINE 29: bl.ip_inet = wl.ip_inet
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ^
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts.
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Merge Logic Stats:
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Created detections: 0
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Cleaned invalid detections: 0
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: Skipped (whitelisted): 0
|
||||
Jan 02 15:40:01 ids.alfacom.it ids-list-fetcher[9493]: ============================================================
|
||||
Jan 02 15:40:01 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully.
|
||||
Jan 02 15:40:01 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service.
|
||||
92
deployment/migrations/008_force_inet_types.sql
Normal file
92
deployment/migrations/008_force_inet_types.sql
Normal file
@ -0,0 +1,92 @@
|
||||
-- Migration 008: Force INET/CIDR types (unconditional)
|
||||
-- Fixes issues where columns remained TEXT after conditional migration 007
|
||||
-- Date: 2026-01-02
|
||||
|
||||
BEGIN;
|
||||
|
||||
-- ============================================================================
|
||||
-- FORCE DROP AND RECREATE ALL INET COLUMNS
|
||||
-- This is unconditional - always executes regardless of current state
|
||||
-- ============================================================================
|
||||
|
||||
-- Drop indexes first (if exist)
|
||||
DROP INDEX IF EXISTS public_blacklist_ip_inet_idx;
|
||||
DROP INDEX IF EXISTS public_blacklist_cidr_inet_idx;
|
||||
DROP INDEX IF EXISTS whitelist_ip_inet_idx;
|
||||
|
||||
-- ============================================================================
|
||||
-- FIX public_blacklist_ips TABLE
|
||||
-- ============================================================================
|
||||
|
||||
-- Drop columns unconditionally
|
||||
ALTER TABLE public_blacklist_ips DROP COLUMN IF EXISTS ip_inet;
|
||||
ALTER TABLE public_blacklist_ips DROP COLUMN IF EXISTS cidr_inet;
|
||||
|
||||
-- Recreate with correct INET/CIDR types
|
||||
ALTER TABLE public_blacklist_ips ADD COLUMN ip_inet inet;
|
||||
ALTER TABLE public_blacklist_ips ADD COLUMN cidr_inet cidr;
|
||||
|
||||
-- Populate from existing text data
|
||||
UPDATE public_blacklist_ips
|
||||
SET
|
||||
ip_inet = CASE
|
||||
WHEN ip_address ~ '/' THEN ip_address::inet
|
||||
ELSE ip_address::inet
|
||||
END,
|
||||
cidr_inet = CASE
|
||||
WHEN cidr_range IS NOT NULL AND cidr_range != '' THEN cidr_range::cidr
|
||||
WHEN ip_address ~ '/' THEN ip_address::cidr
|
||||
ELSE (ip_address || '/32')::cidr
|
||||
END
|
||||
WHERE ip_inet IS NULL;
|
||||
|
||||
-- Create GiST indexes for fast INET/CIDR containment operators
|
||||
CREATE INDEX public_blacklist_ip_inet_idx ON public_blacklist_ips USING gist(ip_inet inet_ops);
|
||||
CREATE INDEX public_blacklist_cidr_inet_idx ON public_blacklist_ips USING gist(cidr_inet inet_ops);
|
||||
|
||||
-- ============================================================================
|
||||
-- FIX whitelist TABLE
|
||||
-- ============================================================================
|
||||
|
||||
-- Drop column unconditionally
|
||||
ALTER TABLE whitelist DROP COLUMN IF EXISTS ip_inet;
|
||||
|
||||
-- Recreate with correct INET type
|
||||
ALTER TABLE whitelist ADD COLUMN ip_inet inet;
|
||||
|
||||
-- Populate from existing text data
|
||||
UPDATE whitelist
|
||||
SET ip_inet = CASE
|
||||
WHEN ip_address ~ '/' THEN ip_address::inet
|
||||
ELSE ip_address::inet
|
||||
END
|
||||
WHERE ip_inet IS NULL;
|
||||
|
||||
-- Create index for whitelist
|
||||
CREATE INDEX whitelist_ip_inet_idx ON whitelist USING gist(ip_inet inet_ops);
|
||||
|
||||
-- ============================================================================
|
||||
-- UPDATE SCHEMA VERSION
|
||||
-- ============================================================================
|
||||
|
||||
UPDATE schema_version SET version = 8, applied_at = NOW() WHERE id = 1;
|
||||
|
||||
COMMIT;
|
||||
|
||||
-- ============================================================================
|
||||
-- VERIFICATION
|
||||
-- ============================================================================
|
||||
|
||||
SELECT 'Migration 008 completed successfully' as status;
|
||||
SELECT version, applied_at FROM schema_version WHERE id = 1;
|
||||
|
||||
-- Verify column types
|
||||
SELECT
|
||||
table_name,
|
||||
column_name,
|
||||
data_type
|
||||
FROM information_schema.columns
|
||||
WHERE
|
||||
(table_name = 'public_blacklist_ips' AND column_name IN ('ip_inet', 'cidr_inet'))
|
||||
OR (table_name = 'whitelist' AND column_name = 'ip_inet')
|
||||
ORDER BY table_name, column_name;
|
||||
@ -70,10 +70,12 @@ export const detections = pgTable("detections", {
|
||||
}));
|
||||
|
||||
// Whitelist per IP fidati
|
||||
// NOTE: ip_inet is INET type in production (managed by SQL migrations)
|
||||
// Drizzle lacks native INET support, so we use text() here
|
||||
export const whitelist = pgTable("whitelist", {
|
||||
id: varchar("id").primaryKey().default(sql`gen_random_uuid()`),
|
||||
ipAddress: text("ip_address").notNull().unique(),
|
||||
ipInet: text("ip_inet"),
|
||||
ipInet: text("ip_inet"), // Actually INET in production - see migration 008
|
||||
comment: text("comment"),
|
||||
reason: text("reason"),
|
||||
createdBy: text("created_by"),
|
||||
@ -156,12 +158,14 @@ export const publicLists = pgTable("public_lists", {
|
||||
}));
|
||||
|
||||
// Public blacklist IPs from external sources
|
||||
// NOTE: ip_inet/cidr_inet are INET/CIDR types in production (managed by SQL migrations)
|
||||
// Drizzle lacks native INET/CIDR support, so we use text() here
|
||||
export const publicBlacklistIps = pgTable("public_blacklist_ips", {
|
||||
id: varchar("id").primaryKey().default(sql`gen_random_uuid()`),
|
||||
ipAddress: text("ip_address").notNull(),
|
||||
cidrRange: text("cidr_range"),
|
||||
ipInet: text("ip_inet"),
|
||||
cidrInet: text("cidr_inet"),
|
||||
ipInet: text("ip_inet"), // Actually INET in production - see migration 008
|
||||
cidrInet: text("cidr_inet"), // Actually CIDR in production - see migration 008
|
||||
listId: varchar("list_id").notNull().references(() => publicLists.id, { onDelete: 'cascade' }),
|
||||
firstSeen: timestamp("first_seen").defaultNow().notNull(),
|
||||
lastSeen: timestamp("last_seen").defaultNow().notNull(),
|
||||
|
||||
Loading…
Reference in New Issue
Block a user