Fix errors in IP detection and merge logic by correcting data types
Addresses type mismatches in `risk_score` handling and INET comparisons within `merge_logic.py`, ensuring correct data insertion and IP range analysis. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: e1f9b236-1e9e-4ac6-a8f7-8ca066dc8467 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/zqNbsxW
This commit is contained in:
marco370
parent
21ff8c0c4b
commit
a311573d0c
@ -0,0 +1,51 @@
|
||||
journalctl -u ids-list-fetcher -n 50 --no-pager
|
||||
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts.
|
||||
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Merge Logic Stats:
|
||||
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Created detections: 0
|
||||
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Cleaned invalid detections: 0
|
||||
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Skipped (whitelisted): 0
|
||||
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: ============================================================
|
||||
Jan 02 16:11:31 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully.
|
||||
Jan 02 16:11:31 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service.
|
||||
Jan 02 16:15:04 ids.alfacom.it systemd[1]: Starting IDS Public Lists Fetcher Service...
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [2026-01-02 16:15:04] PUBLIC LISTS SYNC
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: Found 2 enabled lists
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Downloading Spamhaus from https://www.spamhaus.org/drop/drop_v4.json...
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Downloading AWS from https://ip-ranges.amazonaws.com/ip-ranges.json...
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Parsing Spamhaus...
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Found 1468 IPs, syncing to database...
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] ✓ Spamhaus: +0 -0 ~1468
|
||||
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Parsing AWS...
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: [16:15:05] Found 9548 IPs, syncing to database...
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: [16:15:05] ✓ AWS: +9548 -0 ~0
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: SYNC SUMMARY
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Success: 2/2
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Errors: 0/2
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Total IPs Added: 9548
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Total IPs Removed: 0
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: RUNNING MERGE LOGIC
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ERROR:merge_logic:Failed to sync detections: column "risk_score" is of type numeric but expression is of type text
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: LINE 13: '75',
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ^
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: HINT: You will need to rewrite or cast the expression.
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Traceback (most recent call last):
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: File "/opt/ids/python_ml/merge_logic.py", line 264, in sync_public_blacklist_detections
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: cur.execute("""
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: psycopg2.errors.DatatypeMismatch: column "risk_score" is of type numeric but expression is of type text
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: LINE 13: '75',
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ^
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: HINT: You will need to rewrite or cast the expression.
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Merge Logic Stats:
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Created detections: 0
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Cleaned invalid detections: 0
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Skipped (whitelisted): 0
|
||||
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||
Jan 02 16:15:05 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully.
|
||||
Jan 02 16:15:05 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service.
|
||||
@ -169,17 +169,27 @@ class MergeLogic:
|
||||
INSERT INTO detections (
|
||||
source_ip,
|
||||
risk_score,
|
||||
confidence,
|
||||
anomaly_type,
|
||||
reason,
|
||||
log_count,
|
||||
first_seen,
|
||||
last_seen,
|
||||
detection_source,
|
||||
blacklist_id,
|
||||
detected_at,
|
||||
blocked
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s)
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||
RETURNING id
|
||||
""", (
|
||||
ip_address,
|
||||
str(risk_score),
|
||||
risk_score, # numeric, not string
|
||||
100.0, # confidence
|
||||
'public_blacklist',
|
||||
'IP in public blacklist',
|
||||
1, # log_count
|
||||
datetime.utcnow(), # first_seen
|
||||
datetime.utcnow(), # last_seen
|
||||
'public_blacklist',
|
||||
blacklist_id,
|
||||
datetime.utcnow(),
|
||||
@ -213,6 +223,7 @@ class MergeLogic:
|
||||
try:
|
||||
with conn.cursor() as cur:
|
||||
# Delete detections for IPs in whitelist ranges (CIDR-aware)
|
||||
# Cast both sides to inet explicitly for type safety
|
||||
cur.execute("""
|
||||
DELETE FROM detections d
|
||||
WHERE d.detection_source = 'public_blacklist'
|
||||
@ -221,8 +232,8 @@ class MergeLogic:
|
||||
WHERE wl.active = true
|
||||
AND wl.ip_inet IS NOT NULL
|
||||
AND (
|
||||
d.source_ip::inet = wl.ip_inet
|
||||
OR d.source_ip::inet <<= wl.ip_inet
|
||||
d.source_ip::inet = wl.ip_inet::inet
|
||||
OR d.source_ip::inet <<= wl.ip_inet::inet
|
||||
)
|
||||
)
|
||||
""")
|
||||
@ -265,7 +276,12 @@ class MergeLogic:
|
||||
INSERT INTO detections (
|
||||
source_ip,
|
||||
risk_score,
|
||||
confidence,
|
||||
anomaly_type,
|
||||
reason,
|
||||
log_count,
|
||||
first_seen,
|
||||
last_seen,
|
||||
detection_source,
|
||||
blacklist_id,
|
||||
detected_at,
|
||||
@ -273,8 +289,13 @@ class MergeLogic:
|
||||
)
|
||||
SELECT DISTINCT
|
||||
bl.ip_address,
|
||||
'75',
|
||||
75::numeric,
|
||||
100::numeric,
|
||||
'public_blacklist',
|
||||
'IP in public blacklist',
|
||||
1,
|
||||
NOW(),
|
||||
NOW(),
|
||||
'public_blacklist',
|
||||
bl.id,
|
||||
NOW(),
|
||||
@ -283,14 +304,15 @@ class MergeLogic:
|
||||
WHERE bl.is_active = true
|
||||
AND bl.ip_inet IS NOT NULL
|
||||
-- Priority 1: Exclude if in manual whitelist (highest priority)
|
||||
-- Cast to inet explicitly for type safety
|
||||
AND NOT EXISTS (
|
||||
SELECT 1 FROM whitelist wl
|
||||
WHERE wl.active = true
|
||||
AND wl.source = 'manual'
|
||||
AND wl.ip_inet IS NOT NULL
|
||||
AND (
|
||||
bl.ip_inet = wl.ip_inet
|
||||
OR bl.ip_inet <<= wl.ip_inet
|
||||
bl.ip_inet::inet = wl.ip_inet::inet
|
||||
OR bl.ip_inet::inet <<= wl.ip_inet::inet
|
||||
)
|
||||
)
|
||||
-- Priority 2: Exclude if in public whitelist
|
||||
@ -300,8 +322,8 @@ class MergeLogic:
|
||||
AND wl.source != 'manual'
|
||||
AND wl.ip_inet IS NOT NULL
|
||||
AND (
|
||||
bl.ip_inet = wl.ip_inet
|
||||
OR bl.ip_inet <<= wl.ip_inet
|
||||
bl.ip_inet::inet = wl.ip_inet::inet
|
||||
OR bl.ip_inet::inet <<= wl.ip_inet::inet
|
||||
)
|
||||
)
|
||||
-- Avoid duplicate detections
|
||||
|
||||
Loading…
Reference in New Issue
Block a user