Fix errors in IP detection and merge logic by correcting data types
Addresses type mismatches in `risk_score` handling and INET comparisons within `merge_logic.py`, ensuring correct data insertion and IP range analysis. Replit-Commit-Author: Agent Replit-Commit-Session-Id: 7a657272-55ba-4a79-9a2e-f1ed9bc7a528 Replit-Commit-Checkpoint-Type: full_checkpoint Replit-Commit-Event-Id: e1f9b236-1e9e-4ac6-a8f7-8ca066dc8467 Replit-Commit-Screenshot-Url: https://storage.googleapis.com/screenshot-production-us-central1/449cf7c4-c97a-45ae-8234-e5c5b8d6a84f/7a657272-55ba-4a79-9a2e-f1ed9bc7a528/zqNbsxW
This commit is contained in:
marco370
parent
21ff8c0c4b
commit
a311573d0c
@ -0,0 +1,51 @@
|
|||||||
|
journalctl -u ids-list-fetcher -n 50 --no-pager
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts.
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Merge Logic Stats:
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Created detections: 0
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Cleaned invalid detections: 0
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Skipped (whitelisted): 0
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: ============================================================
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully.
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service.
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it systemd[1]: Starting IDS Public Lists Fetcher Service...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [2026-01-02 16:15:04] PUBLIC LISTS SYNC
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: Found 2 enabled lists
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Downloading Spamhaus from https://www.spamhaus.org/drop/drop_v4.json...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Downloading AWS from https://ip-ranges.amazonaws.com/ip-ranges.json...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Parsing Spamhaus...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Found 1468 IPs, syncing to database...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] ✓ Spamhaus: +0 -0 ~1468
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Parsing AWS...
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: [16:15:05] Found 9548 IPs, syncing to database...
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: [16:15:05] ✓ AWS: +9548 -0 ~0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: SYNC SUMMARY
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Success: 2/2
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Errors: 0/2
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Total IPs Added: 9548
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Total IPs Removed: 0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: RUNNING MERGE LOGIC
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ERROR:merge_logic:Failed to sync detections: column "risk_score" is of type numeric but expression is of type text
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: LINE 13: '75',
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ^
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: HINT: You will need to rewrite or cast the expression.
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Traceback (most recent call last):
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: File "/opt/ids/python_ml/merge_logic.py", line 264, in sync_public_blacklist_detections
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: cur.execute("""
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: psycopg2.errors.DatatypeMismatch: column "risk_score" is of type numeric but expression is of type text
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: LINE 13: '75',
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ^
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: HINT: You will need to rewrite or cast the expression.
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Merge Logic Stats:
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Created detections: 0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Cleaned invalid detections: 0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Skipped (whitelisted): 0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully.
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service.
|
||||||
@ -169,17 +169,27 @@ class MergeLogic:
|
|||||||
INSERT INTO detections (
|
INSERT INTO detections (
|
||||||
source_ip,
|
source_ip,
|
||||||
risk_score,
|
risk_score,
|
||||||
|
confidence,
|
||||||
anomaly_type,
|
anomaly_type,
|
||||||
|
reason,
|
||||||
|
log_count,
|
||||||
|
first_seen,
|
||||||
|
last_seen,
|
||||||
detection_source,
|
detection_source,
|
||||||
blacklist_id,
|
blacklist_id,
|
||||||
detected_at,
|
detected_at,
|
||||||
blocked
|
blocked
|
||||||
) VALUES (%s, %s, %s, %s, %s, %s, %s)
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||||
RETURNING id
|
RETURNING id
|
||||||
""", (
|
""", (
|
||||||
ip_address,
|
ip_address,
|
||||||
str(risk_score),
|
risk_score, # numeric, not string
|
||||||
|
100.0, # confidence
|
||||||
'public_blacklist',
|
'public_blacklist',
|
||||||
|
'IP in public blacklist',
|
||||||
|
1, # log_count
|
||||||
|
datetime.utcnow(), # first_seen
|
||||||
|
datetime.utcnow(), # last_seen
|
||||||
'public_blacklist',
|
'public_blacklist',
|
||||||
blacklist_id,
|
blacklist_id,
|
||||||
datetime.utcnow(),
|
datetime.utcnow(),
|
||||||
@ -213,6 +223,7 @@ class MergeLogic:
|
|||||||
try:
|
try:
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
# Delete detections for IPs in whitelist ranges (CIDR-aware)
|
# Delete detections for IPs in whitelist ranges (CIDR-aware)
|
||||||
|
# Cast both sides to inet explicitly for type safety
|
||||||
cur.execute("""
|
cur.execute("""
|
||||||
DELETE FROM detections d
|
DELETE FROM detections d
|
||||||
WHERE d.detection_source = 'public_blacklist'
|
WHERE d.detection_source = 'public_blacklist'
|
||||||
@ -221,8 +232,8 @@ class MergeLogic:
|
|||||||
WHERE wl.active = true
|
WHERE wl.active = true
|
||||||
AND wl.ip_inet IS NOT NULL
|
AND wl.ip_inet IS NOT NULL
|
||||||
AND (
|
AND (
|
||||||
d.source_ip::inet = wl.ip_inet
|
d.source_ip::inet = wl.ip_inet::inet
|
||||||
OR d.source_ip::inet <<= wl.ip_inet
|
OR d.source_ip::inet <<= wl.ip_inet::inet
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
@ -265,7 +276,12 @@ class MergeLogic:
|
|||||||
INSERT INTO detections (
|
INSERT INTO detections (
|
||||||
source_ip,
|
source_ip,
|
||||||
risk_score,
|
risk_score,
|
||||||
|
confidence,
|
||||||
anomaly_type,
|
anomaly_type,
|
||||||
|
reason,
|
||||||
|
log_count,
|
||||||
|
first_seen,
|
||||||
|
last_seen,
|
||||||
detection_source,
|
detection_source,
|
||||||
blacklist_id,
|
blacklist_id,
|
||||||
detected_at,
|
detected_at,
|
||||||
@ -273,8 +289,13 @@ class MergeLogic:
|
|||||||
)
|
)
|
||||||
SELECT DISTINCT
|
SELECT DISTINCT
|
||||||
bl.ip_address,
|
bl.ip_address,
|
||||||
'75',
|
75::numeric,
|
||||||
|
100::numeric,
|
||||||
'public_blacklist',
|
'public_blacklist',
|
||||||
|
'IP in public blacklist',
|
||||||
|
1,
|
||||||
|
NOW(),
|
||||||
|
NOW(),
|
||||||
'public_blacklist',
|
'public_blacklist',
|
||||||
bl.id,
|
bl.id,
|
||||||
NOW(),
|
NOW(),
|
||||||
@ -283,14 +304,15 @@ class MergeLogic:
|
|||||||
WHERE bl.is_active = true
|
WHERE bl.is_active = true
|
||||||
AND bl.ip_inet IS NOT NULL
|
AND bl.ip_inet IS NOT NULL
|
||||||
-- Priority 1: Exclude if in manual whitelist (highest priority)
|
-- Priority 1: Exclude if in manual whitelist (highest priority)
|
||||||
|
-- Cast to inet explicitly for type safety
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1 FROM whitelist wl
|
SELECT 1 FROM whitelist wl
|
||||||
WHERE wl.active = true
|
WHERE wl.active = true
|
||||||
AND wl.source = 'manual'
|
AND wl.source = 'manual'
|
||||||
AND wl.ip_inet IS NOT NULL
|
AND wl.ip_inet IS NOT NULL
|
||||||
AND (
|
AND (
|
||||||
bl.ip_inet = wl.ip_inet
|
bl.ip_inet::inet = wl.ip_inet::inet
|
||||||
OR bl.ip_inet <<= wl.ip_inet
|
OR bl.ip_inet::inet <<= wl.ip_inet::inet
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
-- Priority 2: Exclude if in public whitelist
|
-- Priority 2: Exclude if in public whitelist
|
||||||
@ -300,8 +322,8 @@ class MergeLogic:
|
|||||||
AND wl.source != 'manual'
|
AND wl.source != 'manual'
|
||||||
AND wl.ip_inet IS NOT NULL
|
AND wl.ip_inet IS NOT NULL
|
||||||
AND (
|
AND (
|
||||||
bl.ip_inet = wl.ip_inet
|
bl.ip_inet::inet = wl.ip_inet::inet
|
||||||
OR bl.ip_inet <<= wl.ip_inet
|
OR bl.ip_inet::inet <<= wl.ip_inet::inet
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
-- Avoid duplicate detections
|
-- Avoid duplicate detections
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user