From a311573d0c3135d8cdb44eda89003b3be333ebcd Mon Sep 17 00:00:00 2001 From: marco370 <48531002-marco370@users.noreply.replit.com> Date: Fri, 2 Jan 2026 15:19:26 +0000 Subject: [PATCH] Fix errors in IP detection and merge logic by correcting data types Addresses type mismatches in `risk_score` handling and INET comparisons within `merge_logic.py`, ensuring correct data insertion and IP range analysis. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: e1f9b236-1e9e-4ac6-a8f7-8ca066dc8467 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/zqNbsxW --- ...-50-no-pager-Jan-02-16-1_1767366961971.txt | 51 +++++++++++++++++++ python_ml/merge_logic.py | 40 +++++++++++---- 2 files changed, 82 insertions(+), 9 deletions(-) create mode 100644 attached_assets/Pasted-journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-16-1_1767366961971.txt diff --git a/attached_assets/Pasted-journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-16-1_1767366961971.txt b/attached_assets/Pasted-journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-16-1_1767366961971.txt new file mode 100644 index 0000000..303b913 --- /dev/null +++ b/attached_assets/Pasted-journalctl-u-ids-list-fetcher-n-50-no-pager-Jan-02-16-1_1767366961971.txt @@ -0,0 +1,51 @@ +journalctl -u ids-list-fetcher -n 50 --no-pager +Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts. +Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Merge Logic Stats: +Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Created detections: 0 +Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Cleaned invalid detections: 0 +Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Skipped (whitelisted): 0 +Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: ============================================================ +Jan 02 16:11:31 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully. +Jan 02 16:11:31 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service. +Jan 02 16:15:04 ids.alfacom.it systemd[1]: Starting IDS Public Lists Fetcher Service... +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: ============================================================ +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [2026-01-02 16:15:04] PUBLIC LISTS SYNC +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: ============================================================ +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: Found 2 enabled lists +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Downloading Spamhaus from https://www.spamhaus.org/drop/drop_v4.json... +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Downloading AWS from https://ip-ranges.amazonaws.com/ip-ranges.json... +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Parsing Spamhaus... +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Found 1468 IPs, syncing to database... +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] ✓ Spamhaus: +0 -0 ~1468 +Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Parsing AWS... +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: [16:15:05] Found 9548 IPs, syncing to database... +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: [16:15:05] ✓ AWS: +9548 -0 ~0 +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================ +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: SYNC SUMMARY +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================ +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Success: 2/2 +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Errors: 0/2 +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Total IPs Added: 9548 +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Total IPs Removed: 0 +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================ +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================ +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: RUNNING MERGE LOGIC +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================ +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ERROR:merge_logic:Failed to sync detections: column "risk_score" is of type numeric but expression is of type text +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: LINE 13: '75', +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ^ +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: HINT: You will need to rewrite or cast the expression. +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Traceback (most recent call last): +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: File "/opt/ids/python_ml/merge_logic.py", line 264, in sync_public_blacklist_detections +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: cur.execute(""" +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: psycopg2.errors.DatatypeMismatch: column "risk_score" is of type numeric but expression is of type text +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: LINE 13: '75', +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ^ +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: HINT: You will need to rewrite or cast the expression. +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Merge Logic Stats: +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Created detections: 0 +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Cleaned invalid detections: 0 +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Skipped (whitelisted): 0 +Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================ +Jan 02 16:15:05 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully. +Jan 02 16:15:05 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service. \ No newline at end of file diff --git a/python_ml/merge_logic.py b/python_ml/merge_logic.py index 00ea653..fb87c53 100755 --- a/python_ml/merge_logic.py +++ b/python_ml/merge_logic.py @@ -169,17 +169,27 @@ class MergeLogic: INSERT INTO detections ( source_ip, risk_score, + confidence, anomaly_type, + reason, + log_count, + first_seen, + last_seen, detection_source, blacklist_id, detected_at, blocked - ) VALUES (%s, %s, %s, %s, %s, %s, %s) + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) RETURNING id """, ( ip_address, - str(risk_score), + risk_score, # numeric, not string + 100.0, # confidence 'public_blacklist', + 'IP in public blacklist', + 1, # log_count + datetime.utcnow(), # first_seen + datetime.utcnow(), # last_seen 'public_blacklist', blacklist_id, datetime.utcnow(), @@ -213,6 +223,7 @@ class MergeLogic: try: with conn.cursor() as cur: # Delete detections for IPs in whitelist ranges (CIDR-aware) + # Cast both sides to inet explicitly for type safety cur.execute(""" DELETE FROM detections d WHERE d.detection_source = 'public_blacklist' @@ -221,8 +232,8 @@ class MergeLogic: WHERE wl.active = true AND wl.ip_inet IS NOT NULL AND ( - d.source_ip::inet = wl.ip_inet - OR d.source_ip::inet <<= wl.ip_inet + d.source_ip::inet = wl.ip_inet::inet + OR d.source_ip::inet <<= wl.ip_inet::inet ) ) """) @@ -265,7 +276,12 @@ class MergeLogic: INSERT INTO detections ( source_ip, risk_score, + confidence, anomaly_type, + reason, + log_count, + first_seen, + last_seen, detection_source, blacklist_id, detected_at, @@ -273,8 +289,13 @@ class MergeLogic: ) SELECT DISTINCT bl.ip_address, - '75', + 75::numeric, + 100::numeric, 'public_blacklist', + 'IP in public blacklist', + 1, + NOW(), + NOW(), 'public_blacklist', bl.id, NOW(), @@ -283,14 +304,15 @@ class MergeLogic: WHERE bl.is_active = true AND bl.ip_inet IS NOT NULL -- Priority 1: Exclude if in manual whitelist (highest priority) + -- Cast to inet explicitly for type safety AND NOT EXISTS ( SELECT 1 FROM whitelist wl WHERE wl.active = true AND wl.source = 'manual' AND wl.ip_inet IS NOT NULL AND ( - bl.ip_inet = wl.ip_inet - OR bl.ip_inet <<= wl.ip_inet + bl.ip_inet::inet = wl.ip_inet::inet + OR bl.ip_inet::inet <<= wl.ip_inet::inet ) ) -- Priority 2: Exclude if in public whitelist @@ -300,8 +322,8 @@ class MergeLogic: AND wl.source != 'manual' AND wl.ip_inet IS NOT NULL AND ( - bl.ip_inet = wl.ip_inet - OR bl.ip_inet <<= wl.ip_inet + bl.ip_inet::inet = wl.ip_inet::inet + OR bl.ip_inet::inet <<= wl.ip_inet::inet ) ) -- Avoid duplicate detections