Compare commits
2 Commits
21ff8c0c4b
...
0298b4a790
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0298b4a790 | ||
|
|
a311573d0c |
@ -0,0 +1,51 @@
|
|||||||
|
journalctl -u ids-list-fetcher -n 50 --no-pager
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: HINT: No operator matches the given name and argument types. You might need to add explicit type casts.
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Merge Logic Stats:
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Created detections: 0
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Cleaned invalid detections: 0
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: Skipped (whitelisted): 0
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it ids-list-fetcher[10401]: ============================================================
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully.
|
||||||
|
Jan 02 16:11:31 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service.
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it systemd[1]: Starting IDS Public Lists Fetcher Service...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [2026-01-02 16:15:04] PUBLIC LISTS SYNC
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: Found 2 enabled lists
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Downloading Spamhaus from https://www.spamhaus.org/drop/drop_v4.json...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Downloading AWS from https://ip-ranges.amazonaws.com/ip-ranges.json...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Parsing Spamhaus...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Found 1468 IPs, syncing to database...
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] ✓ Spamhaus: +0 -0 ~1468
|
||||||
|
Jan 02 16:15:04 ids.alfacom.it ids-list-fetcher[10801]: [16:15:04] Parsing AWS...
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: [16:15:05] Found 9548 IPs, syncing to database...
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: [16:15:05] ✓ AWS: +9548 -0 ~0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: SYNC SUMMARY
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Success: 2/2
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Errors: 0/2
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Total IPs Added: 9548
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Total IPs Removed: 0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: RUNNING MERGE LOGIC
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ERROR:merge_logic:Failed to sync detections: column "risk_score" is of type numeric but expression is of type text
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: LINE 13: '75',
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ^
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: HINT: You will need to rewrite or cast the expression.
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Traceback (most recent call last):
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: File "/opt/ids/python_ml/merge_logic.py", line 264, in sync_public_blacklist_detections
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: cur.execute("""
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: psycopg2.errors.DatatypeMismatch: column "risk_score" is of type numeric but expression is of type text
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: LINE 13: '75',
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ^
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: HINT: You will need to rewrite or cast the expression.
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Merge Logic Stats:
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Created detections: 0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Cleaned invalid detections: 0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: Skipped (whitelisted): 0
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it ids-list-fetcher[10801]: ============================================================
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it systemd[1]: ids-list-fetcher.service: Deactivated successfully.
|
||||||
|
Jan 02 16:15:05 ids.alfacom.it systemd[1]: Finished IDS Public Lists Fetcher Service.
|
||||||
@ -2,7 +2,7 @@
|
|||||||
-- PostgreSQL database dump
|
-- PostgreSQL database dump
|
||||||
--
|
--
|
||||||
|
|
||||||
\restrict bpBxNz70Ka0m0tyhrhacCuMK1bx7vbLdZerc7LHt1LG4ZFdy4h6aJ0zgCj4XJhK
|
\restrict Z0SMaiaV5vhgZwK1NSwbNjjsNLFygVnbAXhqZs1XJQSNOdt4n4ybTuKWgXktCsc
|
||||||
|
|
||||||
-- Dumped from database version 16.11 (74c6bb6)
|
-- Dumped from database version 16.11 (74c6bb6)
|
||||||
-- Dumped by pg_dump version 16.10
|
-- Dumped by pg_dump version 16.10
|
||||||
@ -387,5 +387,5 @@ ALTER TABLE ONLY public.public_blacklist_ips
|
|||||||
-- PostgreSQL database dump complete
|
-- PostgreSQL database dump complete
|
||||||
--
|
--
|
||||||
|
|
||||||
\unrestrict bpBxNz70Ka0m0tyhrhacCuMK1bx7vbLdZerc7LHt1LG4ZFdy4h6aJ0zgCj4XJhK
|
\unrestrict Z0SMaiaV5vhgZwK1NSwbNjjsNLFygVnbAXhqZs1XJQSNOdt4n4ybTuKWgXktCsc
|
||||||
|
|
||||||
|
|||||||
@ -169,17 +169,27 @@ class MergeLogic:
|
|||||||
INSERT INTO detections (
|
INSERT INTO detections (
|
||||||
source_ip,
|
source_ip,
|
||||||
risk_score,
|
risk_score,
|
||||||
|
confidence,
|
||||||
anomaly_type,
|
anomaly_type,
|
||||||
|
reason,
|
||||||
|
log_count,
|
||||||
|
first_seen,
|
||||||
|
last_seen,
|
||||||
detection_source,
|
detection_source,
|
||||||
blacklist_id,
|
blacklist_id,
|
||||||
detected_at,
|
detected_at,
|
||||||
blocked
|
blocked
|
||||||
) VALUES (%s, %s, %s, %s, %s, %s, %s)
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
|
||||||
RETURNING id
|
RETURNING id
|
||||||
""", (
|
""", (
|
||||||
ip_address,
|
ip_address,
|
||||||
str(risk_score),
|
risk_score, # numeric, not string
|
||||||
|
100.0, # confidence
|
||||||
'public_blacklist',
|
'public_blacklist',
|
||||||
|
'IP in public blacklist',
|
||||||
|
1, # log_count
|
||||||
|
datetime.utcnow(), # first_seen
|
||||||
|
datetime.utcnow(), # last_seen
|
||||||
'public_blacklist',
|
'public_blacklist',
|
||||||
blacklist_id,
|
blacklist_id,
|
||||||
datetime.utcnow(),
|
datetime.utcnow(),
|
||||||
@ -213,6 +223,7 @@ class MergeLogic:
|
|||||||
try:
|
try:
|
||||||
with conn.cursor() as cur:
|
with conn.cursor() as cur:
|
||||||
# Delete detections for IPs in whitelist ranges (CIDR-aware)
|
# Delete detections for IPs in whitelist ranges (CIDR-aware)
|
||||||
|
# Cast both sides to inet explicitly for type safety
|
||||||
cur.execute("""
|
cur.execute("""
|
||||||
DELETE FROM detections d
|
DELETE FROM detections d
|
||||||
WHERE d.detection_source = 'public_blacklist'
|
WHERE d.detection_source = 'public_blacklist'
|
||||||
@ -221,8 +232,8 @@ class MergeLogic:
|
|||||||
WHERE wl.active = true
|
WHERE wl.active = true
|
||||||
AND wl.ip_inet IS NOT NULL
|
AND wl.ip_inet IS NOT NULL
|
||||||
AND (
|
AND (
|
||||||
d.source_ip::inet = wl.ip_inet
|
d.source_ip::inet = wl.ip_inet::inet
|
||||||
OR d.source_ip::inet <<= wl.ip_inet
|
OR d.source_ip::inet <<= wl.ip_inet::inet
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
""")
|
""")
|
||||||
@ -265,7 +276,12 @@ class MergeLogic:
|
|||||||
INSERT INTO detections (
|
INSERT INTO detections (
|
||||||
source_ip,
|
source_ip,
|
||||||
risk_score,
|
risk_score,
|
||||||
|
confidence,
|
||||||
anomaly_type,
|
anomaly_type,
|
||||||
|
reason,
|
||||||
|
log_count,
|
||||||
|
first_seen,
|
||||||
|
last_seen,
|
||||||
detection_source,
|
detection_source,
|
||||||
blacklist_id,
|
blacklist_id,
|
||||||
detected_at,
|
detected_at,
|
||||||
@ -273,8 +289,13 @@ class MergeLogic:
|
|||||||
)
|
)
|
||||||
SELECT DISTINCT
|
SELECT DISTINCT
|
||||||
bl.ip_address,
|
bl.ip_address,
|
||||||
'75',
|
75::numeric,
|
||||||
|
100::numeric,
|
||||||
'public_blacklist',
|
'public_blacklist',
|
||||||
|
'IP in public blacklist',
|
||||||
|
1,
|
||||||
|
NOW(),
|
||||||
|
NOW(),
|
||||||
'public_blacklist',
|
'public_blacklist',
|
||||||
bl.id,
|
bl.id,
|
||||||
NOW(),
|
NOW(),
|
||||||
@ -283,14 +304,15 @@ class MergeLogic:
|
|||||||
WHERE bl.is_active = true
|
WHERE bl.is_active = true
|
||||||
AND bl.ip_inet IS NOT NULL
|
AND bl.ip_inet IS NOT NULL
|
||||||
-- Priority 1: Exclude if in manual whitelist (highest priority)
|
-- Priority 1: Exclude if in manual whitelist (highest priority)
|
||||||
|
-- Cast to inet explicitly for type safety
|
||||||
AND NOT EXISTS (
|
AND NOT EXISTS (
|
||||||
SELECT 1 FROM whitelist wl
|
SELECT 1 FROM whitelist wl
|
||||||
WHERE wl.active = true
|
WHERE wl.active = true
|
||||||
AND wl.source = 'manual'
|
AND wl.source = 'manual'
|
||||||
AND wl.ip_inet IS NOT NULL
|
AND wl.ip_inet IS NOT NULL
|
||||||
AND (
|
AND (
|
||||||
bl.ip_inet = wl.ip_inet
|
bl.ip_inet::inet = wl.ip_inet::inet
|
||||||
OR bl.ip_inet <<= wl.ip_inet
|
OR bl.ip_inet::inet <<= wl.ip_inet::inet
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
-- Priority 2: Exclude if in public whitelist
|
-- Priority 2: Exclude if in public whitelist
|
||||||
@ -300,8 +322,8 @@ class MergeLogic:
|
|||||||
AND wl.source != 'manual'
|
AND wl.source != 'manual'
|
||||||
AND wl.ip_inet IS NOT NULL
|
AND wl.ip_inet IS NOT NULL
|
||||||
AND (
|
AND (
|
||||||
bl.ip_inet = wl.ip_inet
|
bl.ip_inet::inet = wl.ip_inet::inet
|
||||||
OR bl.ip_inet <<= wl.ip_inet
|
OR bl.ip_inet::inet <<= wl.ip_inet::inet
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
-- Avoid duplicate detections
|
-- Avoid duplicate detections
|
||||||
|
|||||||
16
version.json
16
version.json
@ -1,7 +1,13 @@
|
|||||||
{
|
{
|
||||||
"version": "1.0.97",
|
"version": "1.0.98",
|
||||||
"lastUpdate": "2026-01-02T14:50:15.450Z",
|
"lastUpdate": "2026-01-02T15:20:02.824Z",
|
||||||
"changelog": [
|
"changelog": [
|
||||||
|
{
|
||||||
|
"version": "1.0.98",
|
||||||
|
"date": "2026-01-02",
|
||||||
|
"type": "patch",
|
||||||
|
"description": "Deployment automatico v1.0.98"
|
||||||
|
},
|
||||||
{
|
{
|
||||||
"version": "1.0.97",
|
"version": "1.0.97",
|
||||||
"date": "2026-01-02",
|
"date": "2026-01-02",
|
||||||
@ -295,12 +301,6 @@
|
|||||||
"date": "2025-11-24",
|
"date": "2025-11-24",
|
||||||
"type": "patch",
|
"type": "patch",
|
||||||
"description": "Deployment automatico v1.0.49"
|
"description": "Deployment automatico v1.0.49"
|
||||||
},
|
|
||||||
{
|
|
||||||
"version": "1.0.48",
|
|
||||||
"date": "2025-11-24",
|
|
||||||
"type": "patch",
|
|
||||||
"description": "Deployment automatico v1.0.48"
|
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
Loading…
Reference in New Issue
Block a user